diff --git a/ChangeLog b/ChangeLog index 0e77cecb..df346f41 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,285 +1,291 @@ ChangeLog +Diff 0.10 to 0.11 + +- Retrieving favicons by evaluation online search's webpage instead of hard-coding + favicon URLs + Diff 0.9.2 to 0.10 - New online search: Semantic Scholar - Use Qt's own QOAuth1 class instead of external library QOAuth - Providing .gitlab-ci.yml to enable continuous integration at GitLab (see https://gitlab.com/tfischer/kbibtex-ci/pipelines) - Greatly refactoring and modernizing CMakeLists.txt files, generation of camel-case headers, private/public linking to libraries, ... - Tabs in the entry editor can show short messages to use, e.g. in which tab DOIs or URLs are to be entered - Updating BibSearch code: cover page improved, preparing code for translations, adding progress bar - Refactoring most regular 'enum's to become 'enum class'es - Preferences class greatly refactored: Based on JSON description, a Python script will generate a C++ class/header pair providing all necessary functions and enums - Cleaning header includes and include guards - Preferring Qt classes over KDE counterparts, e.g. KLineEdit -> QLineEdit or KComboBox -> QComboBox - Migrating many old-style casts like '(int)' to new-style casts like 'static_cast<..>(..)' - Fixing missing tag in XML export - Fixing UI issues with ColorLabelWidget - Preferring QSignalBlocker over manually temporarily disconnecting/disabling signals - Refactoring QSignalMapper into many small lambda-based 'connect's - Refactoring small slot functions into lambda functions - Using KRatingPainter instead of home-made StarRating's paint function - Various improvements and refactoring when (PDF) files get associated with an entry - Having ICU as an optional dependency only, provide internal, static translation from Basic Multilingual Plane (BMP) characters to ASCII-only representations - Adding and extending numerous automated tests - Code modernizations such as using QFontMetrics' 'horizontalAdvance(..)' instead of 'width(..)' - Using QUrl's 'isValid()' instead of 'isEmpty()' - Cleaner code, e.g. using std::numeric_limits::max() instead of magic constant 0x00ffffff - QDebug output uses categories consistently - Across classes, moving code into private subclasses to minimize public interface - Updating translations (contributions by various authors) - Numerous other fixes, clean-ups, refactoring, ... Diff 0.9.1 to 0.9.2 +- Updating favicon URLs - KDE Bug 414375, 414385: Fixing potential crash due to defining TRANSLATION_DOMAIN - KDE Bug 414195: Deselecting all fields on BibTeX Viewer hides the field sorting header Diff 0.9 to 0.9.1 - Fixing GUI issues in preferences/settings - Fixing id extraction during duplicate search - Various minor fixes Diff 0.8.2 to 0.9 - Can be compiled under Windows via Craft - Integrating 'BibSearch', a mobile variant of KBibTeX using QML (Sailfish OS only as of now) - Refactoring of id suggestion editor - Making building Zotero support compile-time optional - Internal refactoring of singleton variables and configuration settings - Adding considerable number of QtText-based tests - Migrating from QRegExp to QRegularExpression class - Using Kate's text editor component for BibTeX sources - Validating user-entered BibTeX sources while typing - More verbose diagnostics while loading BibTeX or RIS data - Various bugfixes in Encoder and BibTeX import classes thanks to improved automated tests - Various small improvements for better robustness, performance, and memory efficiency - KDE Bug 392137: Make entry type (and more fields) available in entry id suggestion setup - KDE Bug 396597: BibLaTeX uses "file" instead of "localfile" - KDE Bugs 405504/406692: Correct handling of ligatures like "st" - Fixing resource leakage as identified by Coverity Scan: CID 325572, 325573 - Integrating commits by Alexander Dunlap, Antonio Rojas, Erik Quaeghebeur, Frederik Schwarzer, Pino Toscano, and Yuri Chornoivan Diff 0.8.1 to 0.8.2 - KDE Bug 388892: Formatting error when saving file ( switching " and } ) - KDE Bug 394659: Crash after compilation - KDE Bug 396343: When saving the file, I am always warned that file has changed in disk - KDE Bug 396598: Bibliography system options contains duplicates - KDE Bug 397027: ScienceDirect search broken - KDE Bug 397604: Untranslated strings from bibtexfields.cpp and bibtexentries.cpp - KDE Bug 398136: KBibTeX crashes when editing element - KDE Bug 401470: Don't remove leading whitespace in macros - Using official APIs for IEEE Xplore and ScienceDirect - Fixing resource leakage as identified by Coverity Scan: CID 287670, 287669 - Fixing issues as identified by clazy - Migrating from HTTP to HTTPS protocol in various places - In encoder classes, migrating away from raw char and char* to Qt classes - Various smaller fixes Diff 0.8 to 0.8.1 - Fixing incorrect version number computation Diff 0.7 to 0.8 Porting from Qt4 to Qt5, from KDE4 (kdelibs) to KDE Frameworks 5, as well as updating various dependencies in the process (e.g. Qt5-based poppler) - Removing old scripts and configuration files - Updating/adding translations - Removing dependency on Qxt as well as sources in src/3rdparty/libqxt - Refactoring various files' location - Various fundamental classes have only optional dependency on KDE Frameworks 5 (default for KDE-based builds, but allows using those classes in Qt5-only setups) - Various modernizations of C++ code towards C++11, including deprecation of SIGNAL/SLOT - Moving bibliography files that previously resided in testset/ into their own Git repository (kbibtex-testset) - Removing ISBNdb as it is no longer a free service - KDE Bug 393032: Updating list of journal abbreviations - KDE Bug 393224: LyX pipe detection (issues with Kile 3 fixed) - KDE Bug 391198: Preview image/vnd.djvu+multipage files - KDE Bug 389306: Removal of libQxt - KDE Bug 387638: Locating correct QtOAuth library fixed - KDE Bug 388688: Screenshots for appdata updated - KDE Bug 386226: Character '~' not recognized in localfile entry - KDE Bug 352517: Invalid report, but more verbose output will be logged - KDE Bug 384741: Wrong ID Reported in Duplicate Dialog - KDE Bug 381119: Do not refer to defunct Gna! anymore (note: Gna! infrastructure shut down before all materials (postings) could be retrieved) - KDE Bug 378497: Fixing crash when closing settings dialog - KDE Bug 368732: More options for ID generation: volume number, first page - Numerous small fixes and changes, run 'git diff v0.6.2..v0.7' for details Contributing authors include: Allen Winter, Andreas Sturmlechner, Andrius Štikonas, Antonio Rojas, Bastien Roucaries, Burkhard Lück, Christoph Feck, Frederik Schwarzer, Joao Carreira, Juergen Spitzmueller, Luigi Toscano, Pino Toscano, Raymond Wooninck, Thomas Fischer, and Yuri Chornoivan Diff 0.6.2 to 0.7 - Dependency on Qt WebKit can be disabled at compile time - New dependency on ICU, used to transliterate text to plain ASCII - Generally improved code quality as detected by code checkers such as Clazy or Coverity - New online search: bioRxiv - Various minor fixes - Search in Zotero is rate limited to avoid overloading server - Using KWallet to store Zotero credentials - Adding basic DBUS support to, for example, open files or paste text Diff 0.6.1 to 0.6.2 - KDE Bug 377401: https://bugs.kde.org/show_bug.cgi?id=377401 KBibTeX fails to load zotero bibliography Diff 0.6 to 0.6.1 - KDE Bug 351455: https://bugs.kde.org/show_bug.cgi?id=351455 Removing soversion from KBibTeX Part - KDE Bug 353898: https://bugs.kde.org/show_bug.cgi?id=353898 Fixing build issues on ARM architecture - KDE Bug 354785: https://bugs.kde.org/show_bug.cgi?id=354785 Using QTextDocument/QTextEdit instead of WebKit/WebEngine: more lightweight and supported on all platforms - Correcting choke on PubMed searches to 10 seconds - Fixing search issues for ACM, Google Scholar, JSTOR, and ScienceDirect - Setting foreground color of colored rows to either black or white for better readability - Disabling OCLC WorldCat (request for support denied by this organization) - Generally improved code quality as detected by code checkers such as Clazy or Coverity - Fixing handling of URLs and their protocols for local files - Fixing setting default id suggestion - Adding 'Keywords' field to .desktop file - Removing file that was licensed under CC BY-NC, but never got installed - Improved Unicode support - Better handling quotation marks and protective curly brackets around titles - Updating translations Diff 0.5.2 to 0.6 - Allowing "unity builds", i.e. merging source code files for faster compilation - Enabling BibUtils support to import/export exotic file formats - Entries can be rated with stars - Adding entry type for Master's thesis - Setting entry identifiers automatically if configured by user - Files (e.g. PDF) can be 'associated' with an entry, including moving/copying/renaming the file to match the bibliography's location and the entry's id - In the element editor, unused tabs are no longer just disabled, but hidden instead - Automatic column-resizing improved - Bibliographies can be imported from Zotero - Adding user interface translations to various languages - New online search engines: CERN Document Server, DOI, IDEAS (RePEc), MR Lookup; fixes to existing search engines - New dockets for file settings, file statistics, and browsing Zotero bibliographies - Value selected in the value list can be added or removed from selected entries - Enhancing the Id Suggestion system - Various fixes as suggested by KDE's code analysis tool Krazy - Numerous small fixes and changes, run 'git diff v0.5.2..v0.6' for details Diff 0.5.1 to 0.5.2 - Migrating to KDE's Git infrastructure - Gna Bug 22418: http://gna.org/bugs/?22418: Relative paths fail to get resolved - KDE Bug 339086: https://bugs.kde.org/show_bug.cgi?id=339086 Fixing ScienceDirect search - KDE Bug 343855: https://bugs.kde.org/show_bug.cgi?id=343855 'Copy Reference' setting in GUI correctly stored - KDE Bug 344495: https://bugs.kde.org/show_bug.cgi?id=344495 Uninitialized variable causes crash - KDE Bug 344497: https://bugs.kde.org/show_bug.cgi?id=344497 Message next to Import button in Search Results - Various minor changes and backports from 0.6.x Run git log v0.5.1..v0.5.2 for a more detailed change log Diff 0.5 to 0.5.1 - KDE Bug 329724: https://bugs.kde.org/show_bug.cgi?id=329724 Fixing sorting issue in main list - KDE Bug 329750: https://bugs.kde.org/show_bug.cgi?id=329750 KBibTeX will set itself as default bibliography editor in KDE - KDE Bug 330700: https://bugs.kde.org/show_bug.cgi?id=330700 Crash when finding PDFs - KDE Bug 332043: https://bugs.kde.org/show_bug.cgi?id=332043 Fixing crash in id suggestion editor - Gentoo Bug 498932: https://bugs.gentoo.org/show_bug.cgi?id=498932 Fixing compilation issue - Gna Bug 21581: http://gna.org/bugs/?21581 Restoring session state (1) - Gna Bug 21545: http://gna.org/bugs/?21545 Restoring session state (2) - Debian Bug 689310: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=689310 Correctly parsing command line arguments if relative paths are given - Various minor clean-ups and improvements Run git log v0.5..v0.5.1 for a more detailed change log Diff 0.4.1 to 0.5 - Initial support for BibLaTeX - Id Suggestion editor like available in old KDE3 version - "Find PDF" function to locate PDF files through web search engines - New online database searches: MathSciNet, Ingenta Connect, Inspire Hep, SAO/NASA ADS, ISBN DB, JStor - Improved memory management - Numerous bug fixes and improvements Diff 0.4 to 0.4.1 - Web searches: Improved support for user-triggered cancelling - SpringerLink search: GUI changes, using api.springer.com for search - PubMed: Limiting search requests per time - ACM Portal: Retrieving "month", fixing HTTP header - JSTOR: fixing HTTP header - Google Scholar: Updates to compensate for changes in Google's web layout; handling redirects - Science Direct: Updates to compensate for changes in Science Direct's web layout; handling redirects - arXiv: Trying to extract bibliographic information from journal strings - BibSonomy: Specifying number of hits to find - Minor changes in IEEExplore search; non-functional due to Qt bug? - Web search uses KDE's proxy settings - Using KDE subsystem to open external files (e.g. PDF files) - Adding preview for images (in addition to PDF or HTML files); handling references to arXiv - Squeezing long file names in user interface - Handling quit actions more gracefully - Improving interface to external programs such as pdflatex - More robust XSL transformations - BibTeX import: guessing encoding information left by JabRef, more informative debug output, improved handling of multiple fields with same name - Reference preview: supporting dark color schemes - Fixing sorting in value list - Fixes in setting color tag to entries - Fixes in name formatting - Keeping user interface read-only for read-only use cases - Numerous bug fixes, closing memory leaks, speed improvements - Fixes in duplicate merging code: remove fields user doesn't want to keep Diff 0.3 to 0.4 - Support for Windows (compiles out of the box) - Configuration file system refactored - Adding more online search engines: SpringerLink, PubMed, ACM Digital Library, JSTOR, IEEE Xplorer, Science Direct - Improving all other online search engines: Google Scholar, arXiv, BibSonomy - "List of Values" refactored, allows to search for items - Introducing preferences dialog to manage various settings - Improved support for drag'n'drop throughout the program - Improving tagging elements with color - Introducing global keyword list to select from - Editing widgets get "history" to select from - Widget for cross references allows to select from existing elements - Introducing duplicate finding and merging code and user interface - Improvements in usability of filter line edit - File view can resize and order columns, settings get stored - Improving file importer and exporter filters - BibTeX references can be sent to LyX via a pipe - Numerous bug fixes diff --git a/src/networking/CMakeLists.txt b/src/networking/CMakeLists.txt index 9cc2df6c..7601f3e4 100644 --- a/src/networking/CMakeLists.txt +++ b/src/networking/CMakeLists.txt @@ -1,161 +1,163 @@ set( kbibtexnetworking_SRCS onlinesearch/onlinesearchabstract.cpp onlinesearch/onlinesearchbibsonomy.cpp onlinesearch/onlinesearcharxiv.cpp onlinesearch/onlinesearchsciencedirect.cpp onlinesearch/onlinesearchgooglescholar.cpp onlinesearch/onlinesearchieeexplore.cpp onlinesearch/onlinesearchpubmed.cpp onlinesearch/onlinesearchacmportal.cpp onlinesearch/onlinesearchspringerlink.cpp onlinesearch/onlinesearchjstor.cpp onlinesearch/onlinesearchmathscinet.cpp onlinesearch/onlinesearchmrlookup.cpp onlinesearch/onlinesearchinspirehep.cpp onlinesearch/onlinesearchcernds.cpp onlinesearch/onlinesearchingentaconnect.cpp onlinesearch/onlinesearchsimplebibtexdownload.cpp onlinesearch/onlinesearchgeneral.cpp onlinesearch/onlinesearchsoanasaads.cpp onlinesearch/onlinesearchideasrepec.cpp onlinesearch/onlinesearchdoi.cpp onlinesearch/onlinesearchbiorxiv.cpp onlinesearch/onlinesearchsemanticscholar.cpp zotero/api.cpp zotero/collectionmodel.cpp zotero/collection.cpp zotero/items.cpp zotero/groups.cpp zotero/oauthwizard.cpp zotero/tags.cpp zotero/tagmodel.cpp associatedfiles.cpp findpdf.cpp + faviconlocator.cpp internalnetworkaccessmanager.cpp urlchecker.cpp ) ecm_qt_declare_logging_category(kbibtexnetworking_SRCS HEADER logging_networking.h IDENTIFIER LOG_KBIBTEX_NETWORKING CATEGORY_NAME kbibtex.networking ) if(UNITY_BUILD) enable_unity_build(kbibtexnetworking kbibtexnetworking_SRCS) endif(UNITY_BUILD) add_library(kbibtexnetworking SHARED ${kbibtexnetworking_SRCS} ) generate_export_header(kbibtexnetworking) add_library(KBibTeX::Networking ALIAS kbibtexnetworking) set_target_properties(kbibtexnetworking PROPERTIES EXPORT_NAME "kbibtexnetworking" VERSION ${KBIBTEX_RELEASE_VERSION} SOVERSION ${KBIBTEX_SOVERSION} ) target_include_directories(kbibtexnetworking INTERFACE $ ) target_link_libraries(kbibtexnetworking PUBLIC Qt5::Core Qt5::Network Qt5::Widgets KBibTeX::Data PRIVATE Poppler::Qt5 Qt5::DBus Qt5::NetworkAuth KF5::ConfigCore KF5::WidgetsAddons KF5::I18n KF5::KIOCore KF5::KIOFileWidgets KBibTeX::Config KBibTeX::Global KBibTeX::IO ) install( TARGETS kbibtexnetworking EXPORT kbibtexnetworking-targets LIBRARY NAMELINK_SKIP ${KDE_INSTALL_TARGETS_DEFAULT_ARGS} ) set_target_properties(kbibtexnetworking PROPERTIES EXPORT_NAME "Networking" ) ecm_generate_headers(kbibtexnetworking_HEADERS HEADER_NAMES AssociatedFiles FindPDF + FavIconLocator InternalNetworkAccessManager UrlChecker onlinesearch/OnlineSearchAbstract onlinesearch/OnlineSearchGeneral onlinesearch/OnlineSearchAcmPortal onlinesearch/OnlineSearchArXiv onlinesearch/OnlineSearchBibsonomy onlinesearch/OnlineSearchBioRxiv onlinesearch/OnlineSearchCERNDS onlinesearch/OnlineSearchDOI onlinesearch/OnlineSearchGoogleScholar onlinesearch/OnlineSearchIDEASRePEc onlinesearch/OnlineSearchIEEEXplore onlinesearch/OnlineSearchIngentaConnect onlinesearch/OnlineSearchInspireHep onlinesearch/OnlineSearchJStor onlinesearch/OnlineSearchMathSciNet onlinesearch/OnlineSearchMRLookup onlinesearch/OnlineSearchPubMed onlinesearch/OnlineSearchScienceDirect onlinesearch/OnlineSearchSemanticScholar onlinesearch/OnlineSearchSimpleBibTeXDownload onlinesearch/OnlineSearchSOANASAADS onlinesearch/OnlineSearchSpringerLink zotero/API zotero/Collection zotero/CollectionModel zotero/Groups zotero/Items zotero/OAuthWizard zotero/TagModel zotero/Tags REQUIRED_HEADERS kbibtexnetworking_HEADERS ) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/kbibtexnetworking_export.h ${kbibtexnetworking_HEADERS} DESTINATION ${KDE_INSTALL_INCLUDEDIR}/KBibTeX/networking COMPONENT Devel ) include(CMakePackageConfigHelpers) write_basic_package_version_file( ${CMAKE_CURRENT_BINARY_DIR}/KBibTeXNetworking-configVersion.cmake VERSION ${PROJECT_VERSION} COMPATIBILITY ExactVersion ) configure_package_config_file(${CMAKE_CURRENT_LIST_DIR}/cmake/KBibTeXNetworking-config.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/KBibTeXNetworking-config.cmake INSTALL_DESTINATION ${KDE_INSTALL_LIBDIR}/cmake/KBibTeX ) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/KBibTeXNetworking-config.cmake ${CMAKE_CURRENT_BINARY_DIR}/KBibTeXNetworking-configVersion.cmake DESTINATION ${KDE_INSTALL_LIBDIR}/cmake/KBibTeX ) diff --git a/src/networking/faviconlocator.cpp b/src/networking/faviconlocator.cpp new file mode 100644 index 00000000..ebae95be --- /dev/null +++ b/src/networking/faviconlocator.cpp @@ -0,0 +1,156 @@ +/*************************************************************************** + * Copyright (C) 2004-2019 by Thomas Fischer * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, see . * + ***************************************************************************/ + +#include "faviconlocator.h" + +#include +#include +#include +#include +#include +#include + +#include "internalnetworkaccessmanager.h" +#include "logging_networking.h" + +FavIconLocator::FavIconLocator(const QUrl &webpageUrl, QObject *parent) + : QObject(parent), favIcon(QIcon::fromTheme(QStringLiteral("applications-internet"))) +{ + static const QRegularExpression invalidChars(QStringLiteral("[^-a-z0-9_]"), QRegularExpression::CaseInsensitiveOption); + static const QString cacheDirectory = QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + QStringLiteral("/favicons/"); + QDir().mkpath(cacheDirectory); + const QString fileNameStem = cacheDirectory + webpageUrl.toDisplayString().remove(invalidChars); + + /// Try to locate icon in cache first before actually querying the webpage + static const QStringList fileNameExtensions {QStringLiteral(".png"), QStringLiteral(".ico")}; + for (const QString &extension : fileNameExtensions) { + const QString fileName = fileNameStem + extension; + const QFileInfo fi(fileName); + if (fi.exists(fileName)) { + if (fi.lastModified().daysTo(QDateTime::currentDateTime()) > 90) { + /// If icon is other than 90 days, delete it and fetch current one + QFile::remove(fileName); + } else { + favIcon = QIcon(fileName); + QTimer::singleShot(100, this, [this]() { + QMetaObject::invokeMethod(this, "gotIcon", Qt::DirectConnection, QGenericReturnArgument(), Q_ARG(QIcon, favIcon)); + }); + return; + } + } + } + + QNetworkRequest request(webpageUrl); + request.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true); + QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); + connect(reply, &QNetworkReply::finished, parent, [this, reply, fileNameStem, webpageUrl]() { + QUrl favIconUrl; + + if (reply->error() == QNetworkReply::NoError) { + /// Assume that favicon information is within the first 4K of HTML code + const QString htmlCode = QString::fromUtf8(reply->readAll()).left(4096); + /// Some ugly but hopefully fast/flexible/robust HTML code parsing + int p1 = -1; + while ((p1 = htmlCode.indexOf(QStringLiteral(" 0) { + const int p2 = htmlCode.indexOf(QChar('>'), p1 + 5); + if (p2 > p1) { + const int p3 = htmlCode.indexOf(QStringLiteral("rel=\""), p1 + 5); + if (p3 > p1 && p3 < p2) { + const int p4 = htmlCode.indexOf(QChar('"'), p3 + 5); + if (p4 > p3 && p4 < p2) { + const QString relValue = htmlCode.mid(p3 + 5, p4 - p3 - 5); + if (relValue == QStringLiteral("icon") || relValue == QStringLiteral("shortcut icon")) { + const int p5 = htmlCode.indexOf(QStringLiteral("href=\""), p1 + 5); + if (p5 > p1 && p5 < p2) { + const int p6 = htmlCode.indexOf(QChar('"'), p5 + 6); + if (p6 > p5 + 5 && p6 < p2) { + QString hrefValue = htmlCode.mid(p5 + 6, p6 - p5 - 6).replace("&", "&").replace(">", ">").replace("<", "<"); + /// Do some resolving in case favicon URL in HTML code is relative + favIconUrl = reply->url().resolved(QUrl(hrefValue)); + if (favIconUrl.isValid()) { + qCDebug(LOG_KBIBTEX_NETWORKING) << "Found favicon URL" << favIconUrl.toDisplayString() << "in HTML code of webpage" << webpageUrl.toDisplayString(); + break; + } else + favIconUrl.clear(); + } + } + } + } + } + } + } + } + + if (!favIconUrl.isValid()) { + favIconUrl = reply->url(); + favIconUrl.setPath(QStringLiteral("/favicon.ico")); + qCInfo(LOG_KBIBTEX_NETWORKING) << "Could not locate favicon in HTML code for webpage" << webpageUrl.toDisplayString() << ", falling back to" << favIconUrl.toDisplayString(); + } + + QNetworkRequest request(favIconUrl); + request.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true); + QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); + connect(reply, &QNetworkReply::finished, this, [this, reply, fileNameStem, favIconUrl, webpageUrl]() { + if (reply->error() == QNetworkReply::NoError) { + const QByteArray iconData = reply->readAll(); + if (iconData.size() > 10) { + QString extension; + if (iconData[1] == 'P' && iconData[2] == 'N' && iconData[3] == 'G') { + /// PNG files have string "PNG" at second to fourth byte + extension = QStringLiteral(".png"); + } else if (iconData[0] == static_cast(0x00) && iconData[1] == static_cast(0x00) && iconData[2] == static_cast(0x01) && iconData[3] == static_cast(0x00)) { + /// Microsoft Icon have first two bytes always 0x0000, + /// third and fourth byte is 0x0001 (for .ico) + extension = QStringLiteral(".ico"); + } else if (iconData[0] == '<') { + /// HTML or XML code + const QString htmlCode = QString::fromUtf8(iconData); + qCWarning(LOG_KBIBTEX_NETWORKING) << "Received XML or HTML data from " << InternalNetworkAccessManager::removeApiKey(reply->url()).toDisplayString() << ": " << htmlCode.left(128); + } else { + qCWarning(LOG_KBIBTEX_NETWORKING) << "Favicon is of unknown format: " << InternalNetworkAccessManager::removeApiKey(reply->url()).toDisplayString(); + } + + if (!extension.isEmpty()) { + const QString filename = fileNameStem + extension; + + QFile iconFile(filename); + if (iconFile.open(QFile::WriteOnly)) { + iconFile.write(iconData); + iconFile.close(); + qCInfo(LOG_KBIBTEX_NETWORKING) << "Got icon from URL" << favIconUrl.toDisplayString() << "for webpage" << webpageUrl.toDisplayString() << "stored in" << filename; + favIcon = QIcon(filename); + } else { + qCWarning(LOG_KBIBTEX_NETWORKING) << "Could not save icon data from URL" << InternalNetworkAccessManager::removeApiKey(reply->url()).toDisplayString() << "to file" << filename; + } + } + } else { + /// Unlikely that an icon's data is less than 10 bytes, + /// must be an error. + qCWarning(LOG_KBIBTEX_NETWORKING) << "Received invalid icon data from " << InternalNetworkAccessManager::removeApiKey(reply->url()).toDisplayString(); + } + } else + qCWarning(LOG_KBIBTEX_NETWORKING) << "Could not download icon from URL " << InternalNetworkAccessManager::removeApiKey(reply->url()).toDisplayString() << ": " << reply->errorString(); + + QMetaObject::invokeMethod(this, "gotIcon", Qt::DirectConnection, QGenericReturnArgument(), Q_ARG(QIcon, favIcon)); + }); + }); +} + +QIcon FavIconLocator::icon() const +{ + return favIcon; +} diff --git a/src/networking/faviconlocator.h b/src/networking/faviconlocator.h new file mode 100644 index 00000000..4de0e7ef --- /dev/null +++ b/src/networking/faviconlocator.h @@ -0,0 +1,69 @@ +/*************************************************************************** + * Copyright (C) 2004-2019 by Thomas Fischer * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, see . * + ***************************************************************************/ + +#ifndef KBIBTEX_NETWORKING_FAVICONLOCATOR_H +#define KBIBTEX_NETWORKING_FAVICONLOCATOR_H + +#include +#include +#include + +/** + * This class tries to locate and download the favicon for a given + * webpage. Upon success, a signal will be triggered. + * At any point in time, the icon can be queried using a function, + * but as long as no specific favicon got retrieved, this function + * will return a generic icon only. + * Favicons located once will be cached on a per-application base. + * + * @author Thomas Fischer + */ +class FavIconLocator : public QObject +{ + Q_OBJECT +public: + /** + * Start searching for a favicon belonging to a given webpage. + * Search will run asynchronous, its end is signalled with + * @see gotIcon(). + * @param webpageUrl Webpage where to search for a favicon + * @param parent QObject-based parent of this object + */ + explicit FavIconLocator(const QUrl &webpageUrl, QObject *parent); + + /** + * Icon know for the webpage this specific object was created + * for. In case no favicon has been downloaded yet (still in + * progress or download failed), then a generic icon will be + * returned. + * + * @return Either the webpage's favicon or a generic icon + */ + QIcon icon() const; + +signals: + /** + * Asynchronous search for a favicon concluded either with the + * retrieved favicon or, in case of any failure, a generic icon. + */ + void gotIcon(QIcon); + +private: + QIcon favIcon; +}; + +#endif // KBIBTEX_NETWORKING_FAVICONLOCATOR_H diff --git a/src/networking/internalnetworkaccessmanager.cpp b/src/networking/internalnetworkaccessmanager.cpp index 25e78a46..fad28db3 100644 --- a/src/networking/internalnetworkaccessmanager.cpp +++ b/src/networking/internalnetworkaccessmanager.cpp @@ -1,245 +1,251 @@ /*************************************************************************** * Copyright (C) 2004-2018 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #include "internalnetworkaccessmanager.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HAVE_KF5 #include #endif // HAVE_KF5 #include "logging_networking.h" /** * @author Thomas Fischer */ class InternalNetworkAccessManager::HTTPEquivCookieJar: public QNetworkCookieJar { Q_OBJECT public: void mergeHtmlHeadCookies(const QString &htmlCode, const QUrl &url) { static const QRegularExpression cookieContent("^([^\"=; ]+)=([^\"=; ]+).*\\bpath=([^\"=; ]+)", QRegularExpression::CaseInsensitiveOption); int p1 = -1; QRegularExpressionMatch cookieContentRegExpMatch; if ((p1 = htmlCode.toLower().indexOf(QStringLiteral("http-equiv=\"set-cookie\""), 0, Qt::CaseInsensitive)) >= 5 && (p1 = htmlCode.lastIndexOf(QStringLiteral("= 0 && (p1 = htmlCode.indexOf(QStringLiteral("content=\""), p1, Qt::CaseInsensitive)) >= 0 && (cookieContentRegExpMatch = cookieContent.match(htmlCode.mid(p1 + 9, 512))).hasMatch()) { const QString key = cookieContentRegExpMatch.captured(1); const QString value = cookieContentRegExpMatch.captured(2); QList cookies = cookiesForUrl(url); cookies.append(QNetworkCookie(key.toLatin1(), value.toLatin1())); setCookiesFromUrl(cookies, url); } } HTTPEquivCookieJar(QObject *parent = nullptr) : QNetworkCookieJar(parent) { /// nothing } }; QString InternalNetworkAccessManager::userAgentString; InternalNetworkAccessManager::InternalNetworkAccessManager(QObject *parent) : QNetworkAccessManager(parent) { cookieJar = new HTTPEquivCookieJar(this); } void InternalNetworkAccessManager::mergeHtmlHeadCookies(const QString &htmlCode, const QUrl &url) { Q_ASSERT_X(cookieJar != nullptr, "void InternalNetworkAccessManager::mergeHtmlHeadCookies(const QString &htmlCode, const QUrl &url)", "cookieJar is invalid"); cookieJar->mergeHtmlHeadCookies(htmlCode, url); setCookieJar(cookieJar); } InternalNetworkAccessManager &InternalNetworkAccessManager::instance() { static InternalNetworkAccessManager self; return self; } QNetworkReply *InternalNetworkAccessManager::get(QNetworkRequest &request, const QUrl &oldUrl) { #ifdef HAVE_KF5 /// Query the KDE subsystem if a proxy has to be used /// for the host of a given URL QString proxyHostName = KProtocolManager::proxyForUrl(request.url()); if (!proxyHostName.isEmpty() && proxyHostName != QStringLiteral("DIRECT")) { /// Extract both hostname and port number for proxy proxyHostName = proxyHostName.mid(proxyHostName.indexOf(QStringLiteral("://")) + 3); QStringList proxyComponents = proxyHostName.split(QStringLiteral(":"), QString::SkipEmptyParts); if (proxyComponents.length() == 1) { /// Proxy configuration is missing a port number, /// using 8080 as default proxyComponents << QStringLiteral("8080"); } if (proxyComponents.length() == 2) { /// Set proxy to Qt's NetworkAccessManager setProxy(QNetworkProxy(QNetworkProxy::HttpProxy, proxyComponents[0], proxyComponents[1].toInt())); } } else { /// No proxy to be used, clear previous settings setProxy(QNetworkProxy()); } #else // HAVE_KF5 setProxy(QNetworkProxy()); #endif // HAVE_KF5 if (!request.hasRawHeader(QByteArray("Accept"))) request.setRawHeader(QByteArray("Accept"), QByteArray("text/*, */*;q=0.7")); request.setRawHeader(QByteArray("Accept-Charset"), QByteArray("utf-8, us-ascii, ISO-8859-1;q=0.7, ISO-8859-15;q=0.7, windows-1252;q=0.3")); request.setRawHeader(QByteArray("Accept-Language"), QByteArray("en-US, en;q=0.9")); + /// Set 'Referer' and 'Origin' to match the request URL's domain, i.e. URL with empty path + QUrl domainUrl = request.url(); + domainUrl.setPath(QString()); + const QByteArray domain = removeApiKey(domainUrl).toDisplayString().toLatin1(); + request.setRawHeader(QByteArray("Referer"), domain); + request.setRawHeader(QByteArray("Origin"), domain); request.setRawHeader(QByteArray("User-Agent"), userAgent().toLatin1()); if (oldUrl.isValid()) request.setRawHeader(QByteArray("Referer"), removeApiKey(oldUrl).toDisplayString().toLatin1()); QNetworkReply *reply = QNetworkAccessManager::get(request); /// Log SSL errors connect(reply, &QNetworkReply::sslErrors, this, &InternalNetworkAccessManager::logSslErrors); return reply; } QNetworkReply *InternalNetworkAccessManager::get(QNetworkRequest &request, const QNetworkReply *oldReply) { return get(request, oldReply == nullptr ? QUrl() : oldReply->url()); } QString InternalNetworkAccessManager::userAgent() { /// Various browser strings to "disguise" origin static const QStringList userAgentList { QStringLiteral("Mozilla/5.0 (Linux; U; Android 2.3.3; en-us; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"), QStringLiteral("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) QupZilla/2.0.2 Chrome/45.0.2454.101 Safari/537.36"), QStringLiteral("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36 OPR/38.0.2220.41"), QStringLiteral("Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"), QStringLiteral("Mozilla/5.0 (compatible; MSIE 9.0; AOL 9.7; AOLBuild 4343.19; Windows NT 6.1; WOW64; Trident/5.0; FunWebProducts)"), QStringLiteral("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.89 Vivaldi/1.0.83.38 Safari/537.36"), QStringLiteral("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.5 (KHTML, like Gecko) YaBrowser/1.0.1084.5402 Chrome/19.0.1084.5402 Safari/536.5"), QStringLiteral("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)"), QStringLiteral("Mozilla/4.0 (compatible; Dillo 2.2)"), QStringLiteral("Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.201.1 Safari/532.0"), QStringLiteral("Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/10.04 Chromium/14.0.813.0 Chrome/14.0.813.0 Safari/535.1"), QStringLiteral("Mozilla/5.0 (X11; U; Linux; de-DE) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.8.0"), QStringLiteral("Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.21 Safari/537.36 MMS/1.0.2459.0"), QStringLiteral("Mozilla/5.0 (X11; U; Linux; C) AppleWebKit/533.3 (KHTML, like Gecko) QtCarBrowser Safari/533.3"), QStringLiteral("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.89 Vivaldi/1.0.83.38 Safari/537.36"), QStringLiteral("Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11"), QStringLiteral("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727) Sleipnir/2.8.4"), QStringLiteral("Mozilla/5.0 (X11; Linux i686; rv:2.2a1pre) Gecko/20110327 SeaMonkey/2.2a1pre"), QStringLiteral("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10"), QStringLiteral("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13) AppleWebKit/603.1.13 (KHTML, like Gecko) Version/10.1 Safari/603.1.13"), QStringLiteral("Mozilla/5.0 (Linux; U; Tizen/1.0 like Android; en-us; AppleWebKit/534.46 (KHTML, like Gecko) Tizen Browser/1.0 Mobile"), QStringLiteral("Emacs-W3/4.0pre.46 URL/p4.0pre.46 (i686-pc-linux; X11)"), QStringLiteral("Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"), QStringLiteral("Lynx/2.8 (compatible; iCab 2.9.8; Macintosh; U; 68K)"), QStringLiteral("Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0"), QStringLiteral("msnbot/2.1"), QStringLiteral("Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10"), QStringLiteral("Mozilla/5.0 (Windows; U; ; en-NZ) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.8.0"), QStringLiteral("NCSA Mosaic/3.0 (Windows 95)"), QStringLiteral("Mozilla/5.0 (SymbianOS/9.1; U; [en]; Series60/3.0 NokiaE60/4.06.0) AppleWebKit/413 (KHTML, like Gecko) Safari/413"), QStringLiteral("Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16") }; if (userAgentString.isEmpty()) { qsrand(time(nullptr)); userAgentString = userAgentList[qrand() % userAgentList.length()]; } return userAgentString; } void InternalNetworkAccessManager::setNetworkReplyTimeout(QNetworkReply *reply, int timeOutSec) { QTimer *timer = new QTimer(reply); connect(timer, &QTimer::timeout, this, &InternalNetworkAccessManager::networkReplyTimeout); m_mapTimerToReply.insert(timer, reply); timer->start(timeOutSec * 1000); connect(reply, &QNetworkReply::finished, this, &InternalNetworkAccessManager::networkReplyFinished); } QString InternalNetworkAccessManager::reverseObfuscate(const QByteArray &a) { if (a.length() % 2 != 0 || a.length() == 0) return QString(); QString result; result.reserve(a.length() / 2); for (int p = a.length() - 1; p >= 0; p -= 2) { const QChar c = QLatin1Char(a.at(p) ^ a.at(p - 1)); result.append(c); } return result; } QUrl InternalNetworkAccessManager::removeApiKey(QUrl url) { QUrlQuery urlQuery(url); urlQuery.removeQueryItem(QStringLiteral("apikey")); urlQuery.removeQueryItem(QStringLiteral("api_key")); url.setQuery(urlQuery); return url; } void InternalNetworkAccessManager::networkReplyTimeout() { QTimer *timer = static_cast(sender()); timer->stop(); QNetworkReply *reply = m_mapTimerToReply[timer]; if (reply != nullptr) { qCWarning(LOG_KBIBTEX_NETWORKING) << "Timeout on reply to " << removeApiKey(reply->url()).toDisplayString(); reply->close(); m_mapTimerToReply.remove(timer); } } void InternalNetworkAccessManager::networkReplyFinished() { QNetworkReply *reply = static_cast(sender()); QTimer *timer = m_mapTimerToReply.key(reply, nullptr); if (timer != nullptr) { disconnect(timer, &QTimer::timeout, this, &InternalNetworkAccessManager::networkReplyTimeout); timer->stop(); m_mapTimerToReply.remove(timer); } } void InternalNetworkAccessManager::logSslErrors(const QList &errors) { QNetworkReply *reply = static_cast(sender()); qCWarning(LOG_KBIBTEX_NETWORKING) << QStringLiteral("Got the following SSL errors when querying the following URL: ") << removeApiKey(reply->url()).toDisplayString(); for (const QSslError &error : errors) qCWarning(LOG_KBIBTEX_NETWORKING) << QStringLiteral(" * ") + error.errorString() << "; Code: " << static_cast(error.error()); } #include "internalnetworkaccessmanager.moc" diff --git a/src/networking/onlinesearch/onlinesearchabstract.cpp b/src/networking/onlinesearch/onlinesearchabstract.cpp index 6290aaef..6844dc97 100644 --- a/src/networking/onlinesearch/onlinesearchabstract.cpp +++ b/src/networking/onlinesearch/onlinesearchabstract.cpp @@ -1,687 +1,606 @@ /*************************************************************************** * Copyright (C) 2004-2019 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #include "onlinesearchabstract.h" #include #include #include #include #include #include #include #include #ifdef HAVE_QTWIDGETS #include #endif // HAVE_QTWIDGETS #ifdef HAVE_KF5 #include #include #endif // HAVE_KF5 #include #include #include "internalnetworkaccessmanager.h" #include "onlinesearchabstract_p.h" +#include "faviconlocator.h" #include "logging_networking.h" const QString OnlineSearchAbstract::queryKeyFreeText = QStringLiteral("free"); const QString OnlineSearchAbstract::queryKeyTitle = QStringLiteral("title"); const QString OnlineSearchAbstract::queryKeyAuthor = QStringLiteral("author"); const QString OnlineSearchAbstract::queryKeyYear = QStringLiteral("year"); const int OnlineSearchAbstract::resultNoError = 0; const int OnlineSearchAbstract::resultCancelled = 0; /// may get redefined in the future! const int OnlineSearchAbstract::resultUnspecifiedError = 1; const int OnlineSearchAbstract::resultAuthorizationRequired = 2; const int OnlineSearchAbstract::resultNetworkError = 3; const int OnlineSearchAbstract::resultInvalidArguments = 4; const char *OnlineSearchAbstract::httpUnsafeChars = "%:/=+$?&\0"; #ifdef HAVE_QTWIDGETS OnlineSearchAbstract::Form::Private::Private() : config(KSharedConfig::openConfig(QStringLiteral("kbibtexrc"))) { /// nothing } QStringList OnlineSearchAbstract::Form::Private::authorLastNames(const Entry &entry) { const Encoder &encoder = Encoder::instance(); const Value v = entry[Entry::ftAuthor]; QStringList result; result.reserve(v.size()); for (const QSharedPointer &vi : v) { QSharedPointer p = vi.dynamicCast(); if (!p.isNull()) result.append(encoder.convertToPlainAscii(p->lastName())); } return result; } QString OnlineSearchAbstract::Form::Private::guessFreeText(const Entry &entry) { /// If there is a DOI value in this entry, use it as free text static const QStringList doiKeys = {Entry::ftDOI, Entry::ftUrl}; for (const QString &doiKey : doiKeys) if (!entry.value(doiKey).isEmpty()) { const QString text = PlainTextValue::text(entry[doiKey]); const QRegularExpressionMatch doiRegExpMatch = KBibTeX::doiRegExp.match(text); if (doiRegExpMatch.hasMatch()) return doiRegExpMatch.captured(0); } /// If there is no free text yet (e.g. no DOI number), try to identify an arXiv eprint number static const QStringList arxivKeys = {QStringLiteral("eprint"), Entry::ftNumber}; for (const QString &arxivKey : arxivKeys) if (!entry.value(arxivKey).isEmpty()) { const QString text = PlainTextValue::text(entry[arxivKey]); const QRegularExpressionMatch arXivRegExpMatch = KBibTeX::arXivRegExpWithPrefix.match(text); if (arXivRegExpMatch.hasMatch()) return arXivRegExpMatch.captured(1); } return QString(); } #endif // HAVE_QTWIDGETS OnlineSearchAbstract::OnlineSearchAbstract(QObject *parent) : QObject(parent), m_hasBeenCanceled(false), numSteps(0), curStep(0), m_previousBusyState(false), m_delayedStoppedSearchReturnCode(0) { m_parent = parent; } #ifdef HAVE_QTWIDGETS QIcon OnlineSearchAbstract::icon(QListWidgetItem *listWidgetItem) { - static const QRegularExpression invalidChars(QStringLiteral("[^-a-z0-9_]"), QRegularExpression::CaseInsensitiveOption); - const QString cacheDirectory = QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + QStringLiteral("/favicons/"); - QDir().mkpath(cacheDirectory); - const QString fileNameStem = cacheDirectory + QString(favIconUrl()).remove(invalidChars); - const QStringList fileNameExtensions {QStringLiteral(".ico"), QStringLiteral(".png"), QString()}; - - for (const QString &extension : fileNameExtensions) { - const QString fileName = fileNameStem + extension; - if (QFileInfo::exists(fileName)) - return QIcon(fileName); - } - - QNetworkRequest request(favIconUrl()); - QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); - reply->setObjectName(fileNameStem); - if (listWidgetItem != nullptr) - m_iconReplyToListWidgetItem.insert(reply, listWidgetItem); - connect(reply, &QNetworkReply::finished, this, &OnlineSearchAbstract::iconDownloadFinished); - return QIcon::fromTheme(QStringLiteral("applications-internet")); + FavIconLocator *fil = new FavIconLocator(homepage(), this); + connect(fil, &FavIconLocator::gotIcon, this, [listWidgetItem](const QIcon &icon) { + listWidgetItem->setIcon(icon); + }); + return fil->icon(); } OnlineSearchAbstract::Form *OnlineSearchAbstract::customWidget(QWidget *) { return nullptr; } void OnlineSearchAbstract::startSearchFromForm() { m_hasBeenCanceled = false; curStep = numSteps = 0; delayedStoppedSearch(resultNoError); } #endif // HAVE_QTWIDGETS QString OnlineSearchAbstract::name() { if (m_name.isEmpty()) { static const QRegularExpression invalidChars(QStringLiteral("[^-a-z0-9]"), QRegularExpression::CaseInsensitiveOption); m_name = label().remove(invalidChars); } return m_name; } bool OnlineSearchAbstract::busy() const { return numSteps > 0 && curStep < numSteps; } void OnlineSearchAbstract::cancel() { m_hasBeenCanceled = true; curStep = numSteps = 0; refreshBusyProperty(); } QStringList OnlineSearchAbstract::splitRespectingQuotationMarks(const QString &text) { int p1 = 0, p2, max = text.length(); QStringList result; while (p1 < max) { while (text[p1] == ' ') ++p1; p2 = p1; if (text[p2] == '"') { ++p2; while (p2 < max && text[p2] != '"') ++p2; } else while (p2 < max && text[p2] != ' ') ++p2; result << text.mid(p1, p2 - p1 + 1).simplified(); p1 = p2 + 1; } return result; } bool OnlineSearchAbstract::handleErrors(QNetworkReply *reply) { QUrl url; return handleErrors(reply, url); } bool OnlineSearchAbstract::handleErrors(QNetworkReply *reply, QUrl &newUrl) { /// The URL to be shown or logged shall not contain any API key const QUrl urlToShow = InternalNetworkAccessManager::removeApiKey(reply->url()); newUrl = QUrl(); if (m_hasBeenCanceled) { stopSearch(resultCancelled); return false; } else if (reply->error() != QNetworkReply::NoError) { m_hasBeenCanceled = true; const QString errorString = reply->errorString(); qCWarning(LOG_KBIBTEX_NETWORKING) << "Search using" << label() << "failed (error code" << reply->error() << "," << errorString << "), HTTP code" << reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() << ":" << reply->attribute(QNetworkRequest::HttpReasonPhraseAttribute).toByteArray() << ") for URL" << urlToShow.toDisplayString(); const QNetworkRequest &request = reply->request(); /// Dump all HTTP headers that were sent with the original request (except for API keys) const QList rawHeadersSent = request.rawHeaderList(); for (const QByteArray &rawHeaderName : rawHeadersSent) { if (rawHeaderName.toLower().contains("apikey") || rawHeaderName.toLower().contains("api-key")) continue; ///< skip dumping header values containing an API key qCDebug(LOG_KBIBTEX_NETWORKING) << "SENT " << rawHeaderName << ":" << request.rawHeader(rawHeaderName); } /// Dump all HTTP headers that were received const QList rawHeadersReceived = reply->rawHeaderList(); for (const QByteArray &rawHeaderName : rawHeadersReceived) { if (rawHeaderName.toLower().contains("apikey") || rawHeaderName.toLower().contains("api-key")) continue; ///< skip dumping header values containing an API key qCDebug(LOG_KBIBTEX_NETWORKING) << "RECVD " << rawHeaderName << ":" << reply->rawHeader(rawHeaderName); } #ifdef HAVE_KF5 sendVisualNotification(errorString.isEmpty() ? i18n("Searching '%1' failed for unknown reason.", label()) : i18n("Searching '%1' failed with error message:\n\n%2", label(), errorString), label(), QStringLiteral("kbibtex"), 7 * 1000); #endif // HAVE_KF5 int resultCode = resultUnspecifiedError; if (reply->error() == QNetworkReply::AuthenticationRequiredError || reply->error() == QNetworkReply::ProxyAuthenticationRequiredError) resultCode = resultAuthorizationRequired; else if (reply->error() == QNetworkReply::HostNotFoundError || reply->error() == QNetworkReply::TimeoutError) resultCode = resultNetworkError; stopSearch(resultCode); return false; } /** * Check the reply for various problems that might point to * more severe issues. Remember: those are only indicators * to problems which have to be handled elsewhere (therefore, * returning 'true' is totally ok here). */ if (reply->attribute(QNetworkRequest::RedirectionTargetAttribute).isValid()) { newUrl = reply->url().resolved(reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl()); } else if (reply->size() == 0) qCWarning(LOG_KBIBTEX_NETWORKING) << "Search using" << label() << "on url" << urlToShow.toDisplayString() << "returned no data"; return true; } QString OnlineSearchAbstract::htmlAttribute(const QString &htmlCode, const int startPos, const QString &attribute) const { const int endPos = htmlCode.indexOf(QLatin1Char('>'), startPos); if (endPos < 0) return QString(); ///< no closing angle bracket found const QString attributePattern = QString(QStringLiteral(" %1=")).arg(attribute); const int attributePatternPos = htmlCode.indexOf(attributePattern, startPos, Qt::CaseInsensitive); if (attributePatternPos < 0 || attributePatternPos > endPos) return QString(); ///< attribute not found within limits const int attributePatternLen = attributePattern.length(); const int openingQuotationMarkPos = attributePatternPos + attributePatternLen; const QChar quotationMark = htmlCode[openingQuotationMarkPos]; if (quotationMark != QLatin1Char('"') && quotationMark != QLatin1Char('\'')) { /// No valid opening quotation mark found int spacePos = openingQuotationMarkPos; while (spacePos < endPos && !htmlCode[spacePos].isSpace()) ++spacePos; if (spacePos > endPos) return QString(); ///< no closing space found return htmlCode.mid(openingQuotationMarkPos, spacePos - openingQuotationMarkPos); } else { /// Attribute has either single or double quotation marks const int closingQuotationMarkPos = htmlCode.indexOf(quotationMark, openingQuotationMarkPos + 1); if (closingQuotationMarkPos < 0 || closingQuotationMarkPos > endPos) return QString(); ///< closing quotation mark not found within limits return htmlCode.mid(openingQuotationMarkPos + 1, closingQuotationMarkPos - openingQuotationMarkPos - 1); } } bool OnlineSearchAbstract::htmlAttributeIsSelected(const QString &htmlCode, const int startPos, const QString &attribute) const { const int endPos = htmlCode.indexOf(QLatin1Char('>'), startPos); if (endPos < 0) return false; ///< no closing angle bracket found const QString attributePattern = QStringLiteral(" ") + attribute; const int attributePatternPos = htmlCode.indexOf(attributePattern, startPos, Qt::CaseInsensitive); if (attributePatternPos < 0 || attributePatternPos > endPos) return false; ///< attribute not found within limits const int attributePatternLen = attributePattern.length(); const QChar nextAfterAttributePattern = htmlCode[attributePatternPos + attributePatternLen]; if (nextAfterAttributePattern.isSpace() || nextAfterAttributePattern == QLatin1Char('>') || nextAfterAttributePattern == QLatin1Char('/')) /// No value given for attribute (old-style HTML), so assuming it means checked/selected return true; else if (nextAfterAttributePattern == QLatin1Char('=')) { /// Expecting value to attribute, so retrieve it and check for 'selected' or 'checked' const QString attributeValue = htmlAttribute(htmlCode, attributePatternPos, attribute).toLower(); return attributeValue == QStringLiteral("selected") || attributeValue == QStringLiteral("checked"); } /// Reaching this point only if HTML code is invalid return false; } #ifdef HAVE_KF5 /** * Display a passive notification popup using the D-Bus interface. * Copied from KDialog with modifications. */ void OnlineSearchAbstract::sendVisualNotification(const QString &text, const QString &title, const QString &icon, int timeout) { static const QString dbusServiceName = QStringLiteral("org.freedesktop.Notifications"); static const QString dbusInterfaceName = QStringLiteral("org.freedesktop.Notifications"); static const QString dbusPath = QStringLiteral("/org/freedesktop/Notifications"); // check if service already exists on plugin instantiation QDBusConnectionInterface *interface = QDBusConnection::sessionBus().interface(); if (interface == nullptr || !interface->isServiceRegistered(dbusServiceName)) { return; } if (timeout <= 0) timeout = 10 * 1000; QDBusMessage m = QDBusMessage::createMethodCall(dbusServiceName, dbusPath, dbusInterfaceName, QStringLiteral("Notify")); const QList args {QStringLiteral("kdialog"), 0U, icon, title, text, QStringList(), QVariantMap(), timeout}; m.setArguments(args); QDBusMessage replyMsg = QDBusConnection::sessionBus().call(m); if (replyMsg.type() == QDBusMessage::ReplyMessage) { if (!replyMsg.arguments().isEmpty()) { return; } // Not displaying any error messages as this is optional for kdialog // and KPassivePopup is a perfectly valid fallback. //else { // qCDebug(LOG_KBIBTEX_NETWORKING) << "Error: received reply with no arguments."; //} } else if (replyMsg.type() == QDBusMessage::ErrorMessage) { //qCDebug(LOG_KBIBTEX_NETWORKING) << "Error: failed to send D-Bus message"; //qCDebug(LOG_KBIBTEX_NETWORKING) << replyMsg; } else { //qCDebug(LOG_KBIBTEX_NETWORKING) << "Unexpected reply type"; } } #endif // HAVE_KF5 QString OnlineSearchAbstract::encodeURL(QString rawText) { const char *cur = httpUnsafeChars; while (*cur != '\0') { rawText = rawText.replace(QChar(*cur), '%' + QString::number(*cur, 16)); ++cur; } rawText = rawText.replace(QLatin1Char(' '), QLatin1Char('+')); return rawText; } QString OnlineSearchAbstract::decodeURL(QString rawText) { static const QRegularExpression mimeRegExp(QStringLiteral("%([0-9A-Fa-f]{2})")); QRegularExpressionMatch mimeRegExpMatch; while ((mimeRegExpMatch = mimeRegExp.match(rawText)).hasMatch()) { bool ok = false; QChar c(mimeRegExpMatch.captured(1).toInt(&ok, 16)); if (ok) rawText = rawText.replace(mimeRegExpMatch.captured(0), c); } rawText = rawText.replace(QStringLiteral("&"), QStringLiteral("&")).replace(QLatin1Char('+'), QStringLiteral(" ")); return rawText; } QMap OnlineSearchAbstract::formParameters(const QString &htmlText, int startPos) const { /// how to recognize HTML tags static const QString formTagEnd = QStringLiteral(""); static const QString inputTagBegin = QStringLiteral(""); static const QString optionTagBegin = QStringLiteral("