diff --git a/src/networking/onlinesearch/onlinesearchbiorxiv.cpp b/src/networking/onlinesearch/onlinesearchbiorxiv.cpp index 237f93fc..3fb1256c 100644 --- a/src/networking/onlinesearch/onlinesearchbiorxiv.cpp +++ b/src/networking/onlinesearch/onlinesearchbiorxiv.cpp @@ -1,191 +1,191 @@ /*************************************************************************** * Copyright (C) 2016-2018 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #include "onlinesearchbiorxiv.h" #include #include #include #include #include #include "internalnetworkaccessmanager.h" #include "logging_networking.h" class OnlineSearchBioRxiv::Private { public: QSet resultPageUrls; explicit Private(OnlineSearchBioRxiv *) { /// nothing } }; -OnlineSearchBioRxiv::OnlineSearchBioRxiv(QWidget *parent) +OnlineSearchBioRxiv::OnlineSearchBioRxiv(QObject *parent) : OnlineSearchAbstract(parent), d(new OnlineSearchBioRxiv::Private(this)) { /// nothing } OnlineSearchBioRxiv::~OnlineSearchBioRxiv() { /// nothing } void OnlineSearchBioRxiv::startSearch(const QMap &query, int numResults) { m_hasBeenCanceled = false; emit progress(curStep = 0, numSteps = numResults * 2 + 1); QString urlText(QString(QStringLiteral("https://www.biorxiv.org/search/numresults:%1 sort:relevance-rank title_flags:match-phrase format_result:standard ")).arg(numResults)); urlText.append(query[queryKeyFreeText]); bool ok = false; int year = query[queryKeyYear].toInt(&ok); if (ok && year >= 1800 && year < 2100) urlText.append(QString(QStringLiteral(" limit_from:%1-01-01 limit_to:%1-12-31")).arg(year)); const QStringList authors = splitRespectingQuotationMarks(query[queryKeyAuthor]); int authorIndex = 1; for (QStringList::ConstIterator it = authors.constBegin(); it != authors.constEnd(); ++it, ++authorIndex) urlText.append(QString(QStringLiteral(" author%1:%2")).arg(authorIndex).arg(QString(*it).replace(QStringLiteral(" "), QStringLiteral("+")))); const QString title = QString(query[queryKeyTitle]).replace(QStringLiteral(" "), QStringLiteral("+")); if (!title.isEmpty()) urlText.append(QString(QStringLiteral(" title:%1")).arg(title)); QNetworkRequest request(QUrl::fromUserInput(urlText)); QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchBioRxiv::resultsPageDone); refreshBusyProperty(); } QString OnlineSearchBioRxiv::label() const { return i18n("bioRxiv"); } QUrl OnlineSearchBioRxiv::homepage() const { return QUrl(QStringLiteral("https://www.biorxiv.org/")); } QString OnlineSearchBioRxiv::favIconUrl() const { return QStringLiteral("https://www.biorxiv.org/sites/default/files/images/favicon.ico"); } void OnlineSearchBioRxiv::resultsPageDone() { emit progress(++curStep, numSteps); QNetworkReply *reply = static_cast(sender()); if (handleErrors(reply)) { /// ensure proper treatment of UTF-8 characters const QString htmlCode = QString::fromUtf8(reply->readAll().constData()); static const QRegularExpression contentRegExp(QStringLiteral("/content/early/[12]\\d{3}/[01]\\d/\\d{2}/\\d+")); QRegularExpressionMatchIterator contentRegExpMatchIt = contentRegExp.globalMatch(htmlCode); while (contentRegExpMatchIt.hasNext()) { const QRegularExpressionMatch contentRegExpMatch = contentRegExpMatchIt.next(); const QUrl url = QUrl(QStringLiteral("https://www.biorxiv.org") + contentRegExpMatch.captured(0)); d->resultPageUrls.insert(url); } if (d->resultPageUrls.isEmpty()) stopSearch(resultNoError); else { const QUrl firstUrl = *d->resultPageUrls.constBegin(); d->resultPageUrls.remove(firstUrl); QNetworkRequest request(firstUrl); QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchBioRxiv::resultPageDone); } } refreshBusyProperty(); } void OnlineSearchBioRxiv::resultPageDone() { emit progress(++curStep, numSteps); QNetworkReply *reply = static_cast(sender()); if (handleErrors(reply)) { /// ensure proper treatment of UTF-8 characters const QString htmlCode = QString::fromUtf8(reply->readAll().constData()); static const QRegularExpression highwireRegExp(QStringLiteral("/highwire/citation/\\d+/bibtext")); const QRegularExpressionMatch highwireRegExpMatch = highwireRegExp.match(htmlCode); if (highwireRegExpMatch.hasMatch()) { const QUrl url = QUrl(QStringLiteral("https://www.biorxiv.org") + highwireRegExpMatch.captured(0)); QNetworkRequest request(url); QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchBioRxiv::bibTeXDownloadDone); } else if (!d->resultPageUrls.isEmpty()) { const QUrl firstUrl = *d->resultPageUrls.constBegin(); d->resultPageUrls.remove(firstUrl); QNetworkRequest request(firstUrl); QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchBioRxiv::resultPageDone); } else stopSearch(resultNoError); } refreshBusyProperty(); } void OnlineSearchBioRxiv::bibTeXDownloadDone() { emit progress(++curStep, numSteps); QNetworkReply *reply = static_cast(sender()); if (handleErrors(reply)) { /// ensure proper treatment of UTF-8 characters const QString bibTeXcode = QString::fromUtf8(reply->readAll().constData()); if (!bibTeXcode.isEmpty()) { FileImporterBibTeX importer(this); File *bibtexFile = importer.fromString(bibTeXcode); if (bibtexFile != nullptr) { for (const auto &element : const_cast(*bibtexFile)) { QSharedPointer entry = element.dynamicCast(); publishEntry(entry); } delete bibtexFile; } else { qCWarning(LOG_KBIBTEX_NETWORKING) << "No valid BibTeX file results returned on request on" << InternalNetworkAccessManager::removeApiKey(reply->url()).toDisplayString(); } } } if (d->resultPageUrls.isEmpty()) stopSearch(resultNoError); else { const QUrl firstUrl = *d->resultPageUrls.constBegin(); d->resultPageUrls.remove(firstUrl); QNetworkRequest request(firstUrl); QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchBioRxiv::resultPageDone); } refreshBusyProperty(); } diff --git a/src/networking/onlinesearch/onlinesearchbiorxiv.h b/src/networking/onlinesearch/onlinesearchbiorxiv.h index 31a88352..522c74a3 100644 --- a/src/networking/onlinesearch/onlinesearchbiorxiv.h +++ b/src/networking/onlinesearch/onlinesearchbiorxiv.h @@ -1,55 +1,55 @@ /*************************************************************************** * Copyright (C) 2016-2019 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #ifndef KBIBTEX_NETWORKING_ONLINESEARCHBIORXIV_H #define KBIBTEX_NETWORKING_ONLINESEARCHBIORXIV_H #include #ifdef HAVE_KF5 #include "kbibtexnetworking_export.h" #endif // HAVE_KF5 /** * @author Thomas Fischer */ class KBIBTEXNETWORKING_EXPORT OnlineSearchBioRxiv : public OnlineSearchAbstract { Q_OBJECT public: - explicit OnlineSearchBioRxiv(QWidget *parent); + explicit OnlineSearchBioRxiv(QObject *parent); ~OnlineSearchBioRxiv() override; void startSearch(const QMap &query, int numResults) override; QString label() const override; QUrl homepage() const override; protected: QString favIconUrl() const override; private slots: void resultsPageDone(); void resultPageDone(); void bibTeXDownloadDone(); private: class Private; Private *const d; }; #endif // KBIBTEX_NETWORKING_ONLINESEARCHBIORXIV_H diff --git a/src/networking/onlinesearch/onlinesearchcernds.cpp b/src/networking/onlinesearch/onlinesearchcernds.cpp index fc498f87..615276d4 100644 --- a/src/networking/onlinesearch/onlinesearchcernds.cpp +++ b/src/networking/onlinesearch/onlinesearchcernds.cpp @@ -1,113 +1,113 @@ /**************************************************************************** * Copyright (C) 2004-2018 by Thomas Fischer * * Copyright (C) 2013 Yngve I. Levinsen * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ****************************************************************************/ #include "onlinesearchcernds.h" #include #include #include "logging_networking.h" -OnlineSearchCERNDS::OnlineSearchCERNDS(QWidget *parent) +OnlineSearchCERNDS::OnlineSearchCERNDS(QObject *parent) : OnlineSearchSimpleBibTeXDownload(parent) { /// nothing } QString OnlineSearchCERNDS::label() const { return i18n("CERN Document Server"); } QUrl OnlineSearchCERNDS::homepage() const { return QUrl(QStringLiteral("http://cds.cern.ch/")); } QString OnlineSearchCERNDS::favIconUrl() const { return QStringLiteral("http://cds.cern.ch/favicon.ico"); } QUrl OnlineSearchCERNDS::buildQueryUrl(const QMap &query, int numResults) { /// Example for a search URL: /// http://cds.cern.ch/search?action_search=Search&sf=&so=d&rm=&sc=0&of=hx&f=&rg=10&ln=en&as=1&m1=a&p1=stone&f1=title&op1=a&m2=a&p2=smith&f2=author&op2=a&m3=a&p3=&f3= /// of=hx asks for BibTeX results /// rg=10 asks for 10 results /// c=CERN+Document+Server or c=Articles+%26+Preprints to limit scope /// Search search argument (X={1,2,3,...}): /// pX search text /// mX a=all words; o=any; e=exact phrase; p=partial phrase; r=regular expression /// opX a=AND; o=OR; n=AND NOT /// fX ""=any field; title; author; reportnumber; year; fulltext /// Build URL QUrl url = QUrl(QStringLiteral("http://cds.cern.ch/search?ln=en&action_search=Search&c=Articles+%26+Preprints&as=1&sf=&so=d&rm=&sc=0&of=hx&f=")); QUrlQuery q(url); /// Set number of expected results q.addQueryItem(QStringLiteral("rg"), QString::number(numResults)); /// Number search arguments int argumentCount = 0; /// add words from "free text" field const QStringList freeTextWords = splitRespectingQuotationMarks(query[queryKeyFreeText]); for (const QString &word : freeTextWords) { ++argumentCount; q.addQueryItem(QString(QStringLiteral("p%1")).arg(argumentCount), word); q.addQueryItem(QString(QStringLiteral("m%1")).arg(argumentCount), QStringLiteral("a")); q.addQueryItem(QString(QStringLiteral("op%1")).arg(argumentCount), QStringLiteral("a")); q.addQueryItem(QString(QStringLiteral("f%1")).arg(argumentCount), QString()); } /// add words from "author" field const QStringList authorWords = splitRespectingQuotationMarks(query[queryKeyAuthor]); for (const QString &word : authorWords) { ++argumentCount; q.addQueryItem(QString(QStringLiteral("p%1")).arg(argumentCount), word); q.addQueryItem(QString(QStringLiteral("m%1")).arg(argumentCount), QStringLiteral("a")); q.addQueryItem(QString(QStringLiteral("op%1")).arg(argumentCount), QStringLiteral("a")); q.addQueryItem(QString(QStringLiteral("f%1")).arg(argumentCount), QStringLiteral("author")); } /// add words from "title" field const QStringList titleWords = splitRespectingQuotationMarks(query[queryKeyTitle]); for (const QString &word : titleWords) { ++argumentCount; q.addQueryItem(QString(QStringLiteral("p%1")).arg(argumentCount), word); q.addQueryItem(QString(QStringLiteral("m%1")).arg(argumentCount), QStringLiteral("a")); q.addQueryItem(QString(QStringLiteral("op%1")).arg(argumentCount), QStringLiteral("a")); q.addQueryItem(QString(QStringLiteral("f%1")).arg(argumentCount), QStringLiteral("title")); } /// add words from "title" field const QString year = query[queryKeyYear]; if (!year.isEmpty()) { ++argumentCount; q.addQueryItem(QString(QStringLiteral("p%1")).arg(argumentCount), year); q.addQueryItem(QString(QStringLiteral("m%1")).arg(argumentCount), QStringLiteral("a")); q.addQueryItem(QString(QStringLiteral("op%1")).arg(argumentCount), QStringLiteral("a")); q.addQueryItem(QString(QStringLiteral("f%1")).arg(argumentCount), QStringLiteral("year")); } url.setQuery(q); return url; } diff --git a/src/networking/onlinesearch/onlinesearchcernds.h b/src/networking/onlinesearch/onlinesearchcernds.h index a5515d9f..dafae5ae 100644 --- a/src/networking/onlinesearch/onlinesearchcernds.h +++ b/src/networking/onlinesearch/onlinesearchcernds.h @@ -1,45 +1,45 @@ /*************************************************************************** * Copyright (C) 2004-2019 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #ifndef KBIBTEX_NETWORKING_ONLINESEARCHCERNDS_H #define KBIBTEX_NETWORKING_ONLINESEARCHCERNDS_H #include #ifdef HAVE_KF5 #include "kbibtexnetworking_export.h" #endif // HAVE_KF5 /** * @author Thomas Fischer */ class KBIBTEXNETWORKING_EXPORT OnlineSearchCERNDS : public OnlineSearchSimpleBibTeXDownload { Q_OBJECT public: - explicit OnlineSearchCERNDS(QWidget *parent); + explicit OnlineSearchCERNDS(QObject *parent); QString label() const override; QUrl homepage() const override; protected: QString favIconUrl() const override; QUrl buildQueryUrl(const QMap &query, int numResults) override; }; #endif // KBIBTEX_NETWORKING_ONLINESEARCHCERNDS_H diff --git a/src/networking/onlinesearch/onlinesearchdoi.cpp b/src/networking/onlinesearch/onlinesearchdoi.cpp index c461b735..7c9827aa 100644 --- a/src/networking/onlinesearch/onlinesearchdoi.cpp +++ b/src/networking/onlinesearch/onlinesearchdoi.cpp @@ -1,252 +1,252 @@ /*************************************************************************** * Copyright (C) 2004-2019 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #include "onlinesearchdoi.h" #ifdef HAVE_QTWIDGETS #include #include #include #endif // HAVE_QTWIDGETS #include #include #include #ifdef HAVE_KF5 #include #include #endif // HAVE_KF5 #include #include #include "internalnetworkaccessmanager.h" #include "onlinesearchabstract_p.h" #include "logging_networking.h" #ifdef HAVE_QTWIDGETS class OnlineSearchDOI::Form : public OnlineSearchAbstract::Form { Q_OBJECT private: QString configGroupName; void loadState() { KConfigGroup configGroup(d->config, configGroupName); lineEditDoiNumber->setText(configGroup.readEntry(QStringLiteral("doiNumber"), QString())); } public: QLineEdit *lineEditDoiNumber; Form(QWidget *widget) : OnlineSearchAbstract::Form(widget), configGroupName(QStringLiteral("Search Engine DOI")) { QGridLayout *layout = new QGridLayout(this); layout->setMargin(0); QLabel *label = new QLabel(i18n("DOI:"), this); layout->addWidget(label, 0, 0, 1, 1); lineEditDoiNumber = new QLineEdit(this); layout->addWidget(lineEditDoiNumber, 0, 1, 1, 1); lineEditDoiNumber->setClearButtonEnabled(true); connect(lineEditDoiNumber, &QLineEdit::returnPressed, this, &OnlineSearchDOI::Form::returnPressed); layout->setRowStretch(1, 100); lineEditDoiNumber->setFocus(Qt::TabFocusReason); loadState(); } bool readyToStart() const override { return !lineEditDoiNumber->text().isEmpty(); } void copyFromEntry(const Entry &entry) override { lineEditDoiNumber->setText(PlainTextValue::text(entry[Entry::ftDOI])); } void saveState() { KConfigGroup configGroup(d->config, configGroupName); configGroup.writeEntry(QStringLiteral("doiNumber"), lineEditDoiNumber->text()); d->config->sync(); } }; #endif // HAVE_QTWIDGETS class OnlineSearchDOI::OnlineSearchDOIPrivate { public: #ifdef HAVE_QTWIDGETS OnlineSearchDOI::Form *form; #endif // HAVE_QTWIDGETS OnlineSearchDOIPrivate(OnlineSearchDOI *parent) #ifdef HAVE_QTWIDGETS : form(nullptr) #endif // HAVE_QTWIDGETS { Q_UNUSED(parent) } #ifdef HAVE_QTWIDGETS QUrl buildQueryUrl() { if (form == nullptr) { qCWarning(LOG_KBIBTEX_NETWORKING) << "Cannot build query url if no form is specified"; return QUrl(); } return QUrl(QStringLiteral("https://dx.doi.org/") + form->lineEditDoiNumber->text()); } #endif // HAVE_QTWIDGETS QUrl buildQueryUrl(const QMap &query, int numResults) { Q_UNUSED(numResults) const QRegularExpressionMatch doiRegExpMatch = KBibTeX::doiRegExp.match(query[queryKeyFreeText]); if (doiRegExpMatch.hasMatch()) { return QUrl(QStringLiteral("https://dx.doi.org/") + doiRegExpMatch.captured(0)); } return QUrl(); } }; -OnlineSearchDOI::OnlineSearchDOI(QWidget *parent) +OnlineSearchDOI::OnlineSearchDOI(QObject *parent) : OnlineSearchAbstract(parent), d(new OnlineSearchDOIPrivate(this)) { // TODO } OnlineSearchDOI::~OnlineSearchDOI() { delete d; } #ifdef HAVE_QTWIDGETS void OnlineSearchDOI::startSearchFromForm() { m_hasBeenCanceled = false; emit progress(curStep = 0, numSteps = 1); const QUrl url = d->buildQueryUrl(); if (url.isValid()) { QNetworkRequest request(url); request.setRawHeader(QByteArray("Accept"), QByteArray("text/bibliography; style=bibtex")); QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchDOI::downloadDone); d->form->saveState(); } else delayedStoppedSearch(resultNoError); refreshBusyProperty(); } #endif // HAVE_QTWIDGETS void OnlineSearchDOI::startSearch(const QMap &query, int numResults) { m_hasBeenCanceled = false; emit progress(curStep = 0, numSteps = 1); const QUrl url = d->buildQueryUrl(query, numResults); if (url.isValid()) { QNetworkRequest request(url); request.setRawHeader(QByteArray("Accept"), QByteArray("text/bibliography; style=bibtex")); QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchDOI::downloadDone); refreshBusyProperty(); } else delayedStoppedSearch(resultNoError); } QString OnlineSearchDOI::label() const { return i18n("DOI"); } #ifdef HAVE_QTWIDGETS OnlineSearchAbstract::Form *OnlineSearchDOI::customWidget(QWidget *parent) { if (d->form == nullptr) d->form = new OnlineSearchDOI::Form(parent); return d->form; } #endif // HAVE_QTWIDGETS QUrl OnlineSearchDOI::homepage() const { return QUrl(QStringLiteral("https://dx.doi.org/")); } QString OnlineSearchDOI::favIconUrl() const { return QStringLiteral("https://dx.doi.org/favicon.ico"); } void OnlineSearchDOI::downloadDone() { emit progress(++curStep, numSteps); QNetworkReply *reply = static_cast(sender()); QUrl redirUrl; if (handleErrors(reply, redirUrl)) { if (redirUrl.isValid()) { /// redirection to another url ++numSteps; QNetworkRequest request(redirUrl); request.setRawHeader(QByteArray("Accept"), QByteArray("text/bibliography; style=bibtex")); QNetworkReply *newReply = InternalNetworkAccessManager::instance().get(request); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(newReply); connect(newReply, &QNetworkReply::finished, this, &OnlineSearchDOI::downloadDone); } else { /// ensure proper treatment of UTF-8 characters const QString bibTeXcode = QString::fromUtf8(reply->readAll().constData()); if (!bibTeXcode.isEmpty()) { FileImporterBibTeX importer(this); File *bibtexFile = importer.fromString(bibTeXcode); bool hasEntries = false; if (bibtexFile != nullptr) { for (const auto &element : const_cast(*bibtexFile)) { QSharedPointer entry = element.dynamicCast(); hasEntries |= publishEntry(entry); } stopSearch(resultNoError); delete bibtexFile; } else { qCWarning(LOG_KBIBTEX_NETWORKING) << "No valid BibTeX file results returned on request on" << InternalNetworkAccessManager::removeApiKey(reply->url()).toDisplayString(); stopSearch(resultUnspecifiedError); } } else { /// returned file is empty stopSearch(resultNoError); } } } refreshBusyProperty(); } #include "onlinesearchdoi.moc" diff --git a/src/networking/onlinesearch/onlinesearchdoi.h b/src/networking/onlinesearch/onlinesearchdoi.h index c2feec37..295125db 100644 --- a/src/networking/onlinesearch/onlinesearchdoi.h +++ b/src/networking/onlinesearch/onlinesearchdoi.h @@ -1,62 +1,62 @@ /*************************************************************************** * Copyright (C) 2004-2019 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #ifndef KBIBTEX_NETWORKING_ONLINESEARCHDOI_H #define KBIBTEX_NETWORKING_ONLINESEARCHDOI_H #include #ifdef HAVE_KF5 #include "kbibtexnetworking_export.h" #endif // HAVE_KF5 /** * @author Thomas Fischer */ class KBIBTEXNETWORKING_EXPORT OnlineSearchDOI : public OnlineSearchAbstract { Q_OBJECT public: - explicit OnlineSearchDOI(QWidget *parent); + explicit OnlineSearchDOI(QObject *parent); ~OnlineSearchDOI() override; #ifdef HAVE_QTWIDGETS void startSearchFromForm() override; #endif // HAVE_QTWIDGETS void startSearch(const QMap &query, int numResults) override; QString label() const override; #ifdef HAVE_QTWIDGETS OnlineSearchAbstract::Form *customWidget(QWidget *parent) override; #endif // HAVE_QTWIDGETS QUrl homepage() const override; protected: QString favIconUrl() const override; private: #ifdef HAVE_QTWIDGETS class Form; #endif // HAVE_QTWIDGETS class OnlineSearchDOIPrivate; OnlineSearchDOIPrivate *d; private slots: void downloadDone(); }; #endif // KBIBTEX_NETWORKING_ONLINESEARCHDOI_H diff --git a/src/networking/onlinesearch/onlinesearchideasrepec.cpp b/src/networking/onlinesearch/onlinesearchideasrepec.cpp index 767f8f69..8ba96f11 100644 --- a/src/networking/onlinesearch/onlinesearchideasrepec.cpp +++ b/src/networking/onlinesearch/onlinesearchideasrepec.cpp @@ -1,263 +1,263 @@ /*************************************************************************** * Copyright (C) 2004-2018 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #include "onlinesearchideasrepec.h" #include #include #include #include #include #include #include #include #include "internalnetworkaccessmanager.h" #include "logging_networking.h" class OnlineSearchIDEASRePEc::OnlineSearchIDEASRePEcPrivate { public: QSet publicationLinks; QUrl buildQueryUrl(const QMap &query, int numResults) { QString urlBase = QStringLiteral("https://ideas.repec.org/cgi-bin/htsearch?cmd=Search%21&form=extended&m=all&fmt=url&wm=wrd&sp=1&sy=1&dt=range"); bool hasFreeText = !query[queryKeyFreeText].isEmpty(); bool hasTitle = !query[queryKeyTitle].isEmpty(); bool hasAuthor = !query[queryKeyAuthor].isEmpty(); bool hasYear = QRegularExpression(QStringLiteral("^(19|20)[0-9]{2}$")).match(query[queryKeyYear]).hasMatch(); QString fieldWF = QStringLiteral("4BFF"); ///< search whole record by default QString fieldQ, fieldDB, fieldDE; if (hasAuthor && !hasFreeText && !hasTitle) { /// If only the author field is used, search explictly for author fieldWF = QStringLiteral("000F"); fieldQ = query[queryKeyAuthor]; } else if (!hasAuthor && !hasFreeText && hasTitle) { /// If only the title field is used, search explictly for title fieldWF = QStringLiteral("00F0"); fieldQ = query[queryKeyTitle]; } else { fieldQ = query[queryKeyFreeText] + QLatin1Char(' ') + query[queryKeyTitle] + QLatin1Char(' ') + query[queryKeyAuthor] + QLatin1Char(' '); } if (hasYear) { fieldDB = QStringLiteral("01/01/") + query[queryKeyYear]; fieldDE = QStringLiteral("31/12/") + query[queryKeyYear]; } QUrl url(urlBase); QUrlQuery q(url); q.addQueryItem(QStringLiteral("ps"), QString::number(numResults)); q.addQueryItem(QStringLiteral("db"), fieldDB); q.addQueryItem(QStringLiteral("de"), fieldDE); q.addQueryItem(QStringLiteral("q"), fieldQ); q.addQueryItem(QStringLiteral("wf"), fieldWF); url.setQuery(q); return url; } }; -OnlineSearchIDEASRePEc::OnlineSearchIDEASRePEc(QWidget *parent) +OnlineSearchIDEASRePEc::OnlineSearchIDEASRePEc(QObject *parent) : OnlineSearchAbstract(parent), d(new OnlineSearchIDEASRePEc::OnlineSearchIDEASRePEcPrivate()) { /// nothing } OnlineSearchIDEASRePEc::~OnlineSearchIDEASRePEc() { delete d; } void OnlineSearchIDEASRePEc::startSearch(const QMap &query, int numResults) { const QUrl url = d->buildQueryUrl(query, numResults); emit progress(curStep = 0, numSteps = 2 * numResults + 1); m_hasBeenCanceled = false; QNetworkRequest request(url); QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchIDEASRePEc::downloadListDone); refreshBusyProperty(); } QString OnlineSearchIDEASRePEc::label() const { return i18n("IDEAS (RePEc)"); } QString OnlineSearchIDEASRePEc::favIconUrl() const { return QStringLiteral("https://ideas.repec.org/favicon.ico"); } QUrl OnlineSearchIDEASRePEc::homepage() const { return QUrl(QStringLiteral("https://ideas.repec.org/")); } void OnlineSearchIDEASRePEc::downloadListDone() { emit progress(++curStep, numSteps); QNetworkReply *reply = static_cast(sender()); QUrl redirUrl; if (handleErrors(reply, redirUrl)) { if (redirUrl.isValid()) { /// redirection to another url ++numSteps; QNetworkRequest request(redirUrl); QNetworkReply *newReply = InternalNetworkAccessManager::instance().get(request); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(newReply); connect(newReply, &QNetworkReply::finished, this, &OnlineSearchIDEASRePEc::downloadListDone); } else { /// ensure proper treatment of UTF-8 characters const QString htmlCode = QString::fromUtf8(reply->readAll().constData()); const int ol1 = htmlCode.indexOf(QStringLiteral(" results for ")); const int ol2 = htmlCode.indexOf(QStringLiteral(""), ol1 + 2); d->publicationLinks.clear(); if (ol1 > 0 && ol2 > ol1) { const QString olHtmlCode = htmlCode.mid(ol1, ol2 - ol1 + 5); static const QRegularExpression publicationLinkRegExp(QStringLiteral("\"/[a-z](/[^\"]+){1,6}[.]html")); QRegularExpressionMatchIterator publicationLinkRegExpMatchIt = publicationLinkRegExp.globalMatch(olHtmlCode); while (publicationLinkRegExpMatchIt.hasNext()) { const QRegularExpressionMatch publicationLinkRegExpMatch = publicationLinkRegExpMatchIt.next(); const QUrl c = reply->url().resolved(QUrl(publicationLinkRegExpMatch.captured().mid(1))); d->publicationLinks.insert(c); } numSteps += 2 * d->publicationLinks.count(); ///< update number of steps } if (d->publicationLinks.isEmpty()) stopSearch(resultNoError); else { QSet::Iterator it = d->publicationLinks.begin(); const QUrl publicationLink = *it; d->publicationLinks.erase(it); QNetworkRequest request = QNetworkRequest(publicationLink); reply = InternalNetworkAccessManager::instance().get(request, reply); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchIDEASRePEc::downloadPublicationDone); } } } refreshBusyProperty(); } void OnlineSearchIDEASRePEc::downloadPublicationDone() { emit progress(++curStep, numSteps); QNetworkReply *reply = static_cast(sender()); if (handleErrors(reply)) { /// ensure proper treatment of UTF-8 characters const QString htmlCode = QString::fromUtf8(reply->readAll().constData()); QString downloadUrl; static const QString downloadFormStart = QStringLiteral("
form = formParameters(htmlCode, htmlCode.indexOf(downloadFormStart, Qt::CaseInsensitive)); downloadUrl = form[QStringLiteral("url")]; } QMap form = formParameters(htmlCode, htmlCode.indexOf(QStringLiteral("::ConstIterator it = form.constBegin(); while (it != form.constEnd()) { if (!body.isEmpty()) body += QLatin1Char('&'); body += it.key() + QLatin1Char('=') + QString(QUrl::toPercentEncoding(it.value())); ++it; } const QUrl url = QUrl(QStringLiteral("https://ideas.repec.org/cgi-bin/refs.cgi")); QNetworkRequest request(url); request.setHeader(QNetworkRequest::ContentTypeHeader, "application/x-www-form-urlencoded"); reply = InternalNetworkAccessManager::instance().post(request, body.toUtf8()); reply->setProperty("downloadurl", QVariant::fromValue(downloadUrl)); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchIDEASRePEc::downloadBibTeXDone); } refreshBusyProperty(); } void OnlineSearchIDEASRePEc::downloadBibTeXDone() { emit progress(++curStep, numSteps); QNetworkReply *reply = static_cast(sender()); const QString downloadUrl = reply->property("downloadurl").toString(); if (handleErrors(reply)) { /// ensure proper treatment of UTF-8 characters const QString bibTeXcode = QString::fromUtf8(reply->readAll().constData()); if (!bibTeXcode.isEmpty()) { FileImporterBibTeX importer(this); File *bibtexFile = importer.fromString(bibTeXcode); if (bibtexFile != nullptr) { for (const auto &element : const_cast(*bibtexFile)) { QSharedPointer entry = element.dynamicCast(); if (!entry.isNull()) { if (!downloadUrl.isEmpty()) { /// There is an external document associated with this BibTeX entry Value urlValue = entry->value(Entry::ftUrl); urlValue.append(QSharedPointer(new VerbatimText(downloadUrl))); entry->insert(Entry::ftUrl, urlValue); } Value v; v.append(QSharedPointer(new VerbatimText(label()))); entry->insert(QStringLiteral("x-fetchedfrom"), v); emit foundEntry(entry); } } delete bibtexFile; } } if (d->publicationLinks.isEmpty()) stopSearch(resultNoError); else { QSet::Iterator it = d->publicationLinks.begin(); const QUrl publicationLink = *it; d->publicationLinks.erase(it); QNetworkRequest request = QNetworkRequest(publicationLink); reply = InternalNetworkAccessManager::instance().get(request, reply); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchIDEASRePEc::downloadPublicationDone); } } refreshBusyProperty(); } diff --git a/src/networking/onlinesearch/onlinesearchideasrepec.h b/src/networking/onlinesearch/onlinesearchideasrepec.h index 639e45df..29d028b5 100644 --- a/src/networking/onlinesearch/onlinesearchideasrepec.h +++ b/src/networking/onlinesearch/onlinesearchideasrepec.h @@ -1,55 +1,55 @@ /*************************************************************************** * Copyright (C) 2004-2019 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #ifndef KBIBTEX_NETWORKING_ONLINESEARCHIDEASREPEC_H #define KBIBTEX_NETWORKING_ONLINESEARCHIDEASREPEC_H #include #ifdef HAVE_KF5 #include "kbibtexnetworking_export.h" #endif // HAVE_KF5 /** * @author Thomas Fischer */ class KBIBTEXNETWORKING_EXPORT OnlineSearchIDEASRePEc : public OnlineSearchAbstract { Q_OBJECT public: - explicit OnlineSearchIDEASRePEc(QWidget *parent); + explicit OnlineSearchIDEASRePEc(QObject *parent); ~OnlineSearchIDEASRePEc() override; void startSearch(const QMap &query, int numResults) override; QString label() const override; QUrl homepage() const override; protected: QString favIconUrl() const override; private slots: void downloadListDone(); void downloadPublicationDone(); void downloadBibTeXDone(); private: class OnlineSearchIDEASRePEcPrivate; OnlineSearchIDEASRePEcPrivate *d; }; #endif // KBIBTEX_NETWORKING_ONLINESEARCHIDEASREPEC_H diff --git a/src/networking/onlinesearch/onlinesearchinspirehep.cpp b/src/networking/onlinesearch/onlinesearchinspirehep.cpp index 565fc578..1ae81681 100644 --- a/src/networking/onlinesearch/onlinesearchinspirehep.cpp +++ b/src/networking/onlinesearch/onlinesearchinspirehep.cpp @@ -1,83 +1,83 @@ /*************************************************************************** * Copyright (C) 2004-2017 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #include "onlinesearchinspirehep.h" #include -OnlineSearchInspireHep::OnlineSearchInspireHep(QWidget *parent) +OnlineSearchInspireHep::OnlineSearchInspireHep(QObject *parent) : OnlineSearchSimpleBibTeXDownload(parent) { /// nothing } QString OnlineSearchInspireHep::label() const { return i18n("Inspire High-Energy Physics Literature Database"); } QUrl OnlineSearchInspireHep::homepage() const { return QUrl(QStringLiteral("http://inspirehep.net/")); } QString OnlineSearchInspireHep::favIconUrl() const { return QStringLiteral("http://inspirehep.net/favicon.ico"); } QUrl OnlineSearchInspireHep::buildQueryUrl(const QMap &query, int numResults) { static const QString typedSearch = QStringLiteral("%1 %2"); ///< no quotation marks for search term? const QStringList freeTextWords = splitRespectingQuotationMarks(query[queryKeyFreeText]); const QStringList yearWords = splitRespectingQuotationMarks(query[queryKeyYear]); const QStringList titleWords = splitRespectingQuotationMarks(query[queryKeyTitle]); const QStringList authorWords = splitRespectingQuotationMarks(query[queryKeyAuthor]); /// append search terms QStringList queryFragments; queryFragments.reserve(freeTextWords.size() + yearWords.size() + titleWords.size() + authorWords.size()); /// add words from "free text" field for (const QString &text : freeTextWords) queryFragments.append(typedSearch.arg(QStringLiteral("ft"), text)); /// add words from "year" field for (const QString &text : yearWords) queryFragments.append(typedSearch.arg(QStringLiteral("d"), text)); /// add words from "title" field for (const QString &text : titleWords) queryFragments.append(typedSearch.arg(QStringLiteral("t"), text)); /// add words from "author" field for (const QString &text : authorWords) queryFragments.append(typedSearch.arg(QStringLiteral("a"), text)); /// Build URL QString urlText = QStringLiteral("http://inspirehep.net/search?ln=en&ln=en&of=hx&action_search=Search&sf=&so=d&rm=&sc=0"); /// Set number of expected results urlText.append(QString(QStringLiteral("&rg=%1")).arg(numResults)); /// Append actual query urlText.append(QStringLiteral("&p=")); urlText.append(queryFragments.join(QStringLiteral(" and "))); /// URL-encode text urlText = urlText.replace(QLatin1Char(' '), QStringLiteral("%20")).replace(QLatin1Char('"'), QStringLiteral("%22")); return QUrl(urlText); } diff --git a/src/networking/onlinesearch/onlinesearchinspirehep.h b/src/networking/onlinesearch/onlinesearchinspirehep.h index 6e32d8f1..6af975f1 100644 --- a/src/networking/onlinesearch/onlinesearchinspirehep.h +++ b/src/networking/onlinesearch/onlinesearchinspirehep.h @@ -1,45 +1,45 @@ /*************************************************************************** * Copyright (C) 2004-2019 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #ifndef KBIBTEX_NETWORKING_ONLINESEARCHINSPIREHEP_H #define KBIBTEX_NETWORKING_ONLINESEARCHINSPIREHEP_H #include #ifdef HAVE_KF5 #include "kbibtexnetworking_export.h" #endif // HAVE_KF5 /** * @author Thomas Fischer */ class KBIBTEXNETWORKING_EXPORT OnlineSearchInspireHep : public OnlineSearchSimpleBibTeXDownload { Q_OBJECT public: - explicit OnlineSearchInspireHep(QWidget *parent); + explicit OnlineSearchInspireHep(QObject *parent); QString label() const override; QUrl homepage() const override; protected: QString favIconUrl() const override; QUrl buildQueryUrl(const QMap &query, int numResults) override; }; #endif // KBIBTEX_NETWORKING_ONLINESEARCHINSPIREHEP_H diff --git a/src/networking/onlinesearch/onlinesearchmathscinet.cpp b/src/networking/onlinesearch/onlinesearchmathscinet.cpp index 89d399bb..4cf85a5f 100644 --- a/src/networking/onlinesearch/onlinesearchmathscinet.cpp +++ b/src/networking/onlinesearch/onlinesearchmathscinet.cpp @@ -1,258 +1,258 @@ /*************************************************************************** * Copyright (C) 2004-2018 by Thomas Fischer * * 2018 Alexander Dunlap * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #include "onlinesearchmathscinet.h" #include #include #include #include #include #include #include #include #include #include "internalnetworkaccessmanager.h" #include "logging_networking.h" class OnlineSearchMathSciNet::OnlineSearchMathSciNetPrivate { public: QMap queryParameters; int numResults; static const QString queryFormUrl; static const QString queryUrlStem; OnlineSearchMathSciNetPrivate(OnlineSearchMathSciNet *parent) : numResults(0) { Q_UNUSED(parent) } }; const QString OnlineSearchMathSciNet::OnlineSearchMathSciNetPrivate::queryFormUrl = QStringLiteral("https://mathscinet.ams.org/mathscinet/"); const QString OnlineSearchMathSciNet::OnlineSearchMathSciNetPrivate::queryUrlStem = QStringLiteral("https://mathscinet.ams.org/mathscinet/search/publications.html?client=KBibTeX"); -OnlineSearchMathSciNet::OnlineSearchMathSciNet(QWidget *parent) +OnlineSearchMathSciNet::OnlineSearchMathSciNet(QObject *parent) : OnlineSearchAbstract(parent), d(new OnlineSearchMathSciNetPrivate(this)) { /// nothing } OnlineSearchMathSciNet::~OnlineSearchMathSciNet() { delete d; } void OnlineSearchMathSciNet::startSearch(const QMap &query, int numResults) { m_hasBeenCanceled = false; emit progress(curStep = 0, numSteps = 3); d->queryParameters.clear(); d->numResults = qMin(50, numResults); /// limit query to max 50 elements int index = 1; const QString freeText = query[queryKeyFreeText]; const QStringList elementsFreeText = splitRespectingQuotationMarks(freeText); for (const QString &element : elementsFreeText) { d->queryParameters.insert(QString(QStringLiteral("pg%1")).arg(index), QStringLiteral("ALLF")); d->queryParameters.insert(QString(QStringLiteral("s%1")).arg(index), element); ++index; } const QString title = query[queryKeyTitle]; const QStringList elementsTitle = splitRespectingQuotationMarks(title); for (const QString &element : elementsTitle) { d->queryParameters.insert(QString(QStringLiteral("pg%1")).arg(index), QStringLiteral("TI")); d->queryParameters.insert(QString(QStringLiteral("s%1")).arg(index), element); ++index; } const QString authors = query[queryKeyAuthor]; const QStringList elementsAuthor = splitRespectingQuotationMarks(authors); for (const QString &element : elementsAuthor) { d->queryParameters.insert(QString(QStringLiteral("pg%1")).arg(index), QStringLiteral("ICN")); d->queryParameters.insert(QString(QStringLiteral("s%1")).arg(index), element); ++index; } const QString year = query[queryKeyYear]; if (year.isEmpty()) { d->queryParameters.insert(QStringLiteral("dr"), QStringLiteral("all")); } else { d->queryParameters.insert(QStringLiteral("dr"), QStringLiteral("pubyear")); d->queryParameters.insert(QStringLiteral("yrop"), QStringLiteral("eq")); d->queryParameters.insert(QStringLiteral("arg3"), year); } /// issue request for start page QNetworkRequest request(OnlineSearchMathSciNetPrivate::queryFormUrl); QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchMathSciNet::doneFetchingQueryForm); refreshBusyProperty(); } QString OnlineSearchMathSciNet::label() const { return i18n("MathSciNet"); } QString OnlineSearchMathSciNet::favIconUrl() const { return QStringLiteral("http://www.ams.org/favicon.ico"); } QUrl OnlineSearchMathSciNet::homepage() const { return QUrl(QStringLiteral("https://mathscinet.ams.org/mathscinet/help/about.html")); } void OnlineSearchMathSciNet::doneFetchingQueryForm() { QNetworkReply *reply = static_cast(sender()); emit progress(++curStep, numSteps); if (handleErrors(reply)) { // UNUSED const QString htmlText = QString::fromUtf8(reply->readAll().constData()); /// extract form's parameters ... QMap formParams; /// ... and overwrite them with the query's parameters for (QMap::ConstIterator it = d->queryParameters.constBegin(); it != d->queryParameters.constEnd(); ++it) formParams.insert(it.key(), it.value()); /// build url by appending parameters QUrl url(OnlineSearchMathSciNetPrivate::queryUrlStem); QUrlQuery query(url); for (QMap::ConstIterator it = formParams.constBegin(); it != formParams.constEnd(); ++it) query.addQueryItem(it.key(), it.value()); for (int i = 1; i <= d->queryParameters.count(); ++i) query.addQueryItem(QString(QStringLiteral("co%1")).arg(i), QStringLiteral("AND")); ///< join search terms with an AND operation url.setQuery(query); /// issue request for result page QNetworkRequest request(url); QNetworkReply *newReply = InternalNetworkAccessManager::instance().get(request, reply); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(newReply); connect(newReply, &QNetworkReply::finished, this, &OnlineSearchMathSciNet::doneFetchingResultPage); } refreshBusyProperty(); } void OnlineSearchMathSciNet::doneFetchingResultPage() { QNetworkReply *reply = static_cast(sender()); emit progress(++curStep, numSteps); if (handleErrors(reply)) { const QString htmlText = QString::fromUtf8(reply->readAll().constData()); /// extract form's parameters ... QMap formParams = formParameters(htmlText, htmlText.indexOf(QStringLiteral("numResults && checkBoxRegExpMatchIt.hasNext()) { const QRegularExpressionMatch checkBoxRegExpMatch = checkBoxRegExpMatchIt.next(); query.addQueryItem(QStringLiteral("b"), checkBoxRegExpMatch.captured(1)); ++count; } url.setQuery(query); if (count > 0) { /// issue request for bibtex code QNetworkRequest request(url); QNetworkReply *newReply = InternalNetworkAccessManager::instance().get(request, reply); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(newReply); connect(newReply, &QNetworkReply::finished, this, &OnlineSearchMathSciNet::doneFetchingBibTeXcode); } else { /// nothing found stopSearch(resultNoError); } } refreshBusyProperty(); } void OnlineSearchMathSciNet::doneFetchingBibTeXcode() { QNetworkReply *reply = static_cast(sender()); emit progress(curStep = numSteps, numSteps); if (handleErrors(reply)) { /// ensure proper treatment of UTF-8 characters QString htmlCode = QString::fromUtf8(reply->readAll().constData()); QString bibtexCode; int p1 = -1, p2 = -1; while ((p1 = htmlCode.indexOf(QStringLiteral("
"), p2 + 1)) >= 0 && (p2 = htmlCode.indexOf(QStringLiteral("
"), p1 + 1)) >= 0) { bibtexCode += htmlCode.midRef(p1 + 5, p2 - p1 - 5); bibtexCode += QLatin1Char('\n'); } FileImporterBibTeX importer(this); const File *bibtexFile = importer.fromString(bibtexCode); bool hasEntry = false; if (bibtexFile != nullptr) { for (const auto &element : *bibtexFile) { QSharedPointer entry = element.dynamicCast(); hasEntry |= publishEntry(entry); } delete bibtexFile; } stopSearch(hasEntry ? resultNoError : resultUnspecifiedError); } refreshBusyProperty(); } void OnlineSearchMathSciNet::sanitizeEntry(QSharedPointer entry) { OnlineSearchAbstract::sanitizeEntry(entry); const QString ftFJournal = QStringLiteral("fjournal"); if (entry->contains(ftFJournal)) { Value v = entry->value(ftFJournal); entry->remove(Entry::ftJournal); entry->remove(ftFJournal); entry->insert(Entry::ftJournal, v); } } diff --git a/src/networking/onlinesearch/onlinesearchmathscinet.h b/src/networking/onlinesearch/onlinesearchmathscinet.h index 02d0a88e..2478a06f 100644 --- a/src/networking/onlinesearch/onlinesearchmathscinet.h +++ b/src/networking/onlinesearch/onlinesearchmathscinet.h @@ -1,57 +1,57 @@ /*************************************************************************** * Copyright (C) 2004-2019 by Thomas Fischer * * 2018 Alexander Dunlap * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #ifndef KBIBTEX_NETWORKING_ONLINESEARCHMATHSCINET_H #define KBIBTEX_NETWORKING_ONLINESEARCHMATHSCINET_H #include #ifdef HAVE_KF5 #include "kbibtexnetworking_export.h" #endif // HAVE_KF5 /** * @author Thomas Fischer */ class KBIBTEXNETWORKING_EXPORT OnlineSearchMathSciNet : public OnlineSearchAbstract { Q_OBJECT public: - explicit OnlineSearchMathSciNet(QWidget *parent); + explicit OnlineSearchMathSciNet(QObject *parent); ~OnlineSearchMathSciNet() override; void startSearch(const QMap &query, int numResults) override; QString label() const override; QUrl homepage() const override; protected: QString favIconUrl() const override; void sanitizeEntry(QSharedPointer entry) override; private slots: void doneFetchingQueryForm(); void doneFetchingResultPage(); void doneFetchingBibTeXcode(); private: class OnlineSearchMathSciNetPrivate; OnlineSearchMathSciNetPrivate *d; }; #endif // KBIBTEX_NETWORKING_ONLINESEARCHMATHSCINET_H diff --git a/src/networking/onlinesearch/onlinesearchmrlookup.cpp b/src/networking/onlinesearch/onlinesearchmrlookup.cpp index 5abfbbba..44aa6a11 100644 --- a/src/networking/onlinesearch/onlinesearchmrlookup.cpp +++ b/src/networking/onlinesearch/onlinesearchmrlookup.cpp @@ -1,143 +1,143 @@ /*************************************************************************** * Copyright (C) 2004-2018 by Thomas Fischer * * 2014 Pavel Zorin-Kranich * * 2018 Alexander Dunlap * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #include "onlinesearchmrlookup.h" #include #include #include #include #include #include #include #include #include "internalnetworkaccessmanager.h" #include "logging_networking.h" const QString OnlineSearchMRLookup::queryUrlStem = QStringLiteral("https://mathscinet.ams.org/mrlookup"); -OnlineSearchMRLookup::OnlineSearchMRLookup(QWidget *parent) +OnlineSearchMRLookup::OnlineSearchMRLookup(QObject *parent) : OnlineSearchAbstract(parent) { /// nothing } void OnlineSearchMRLookup::startSearch(const QMap &query, int) { m_hasBeenCanceled = false; emit progress(curStep = 0, numSteps = 1); QUrl url(queryUrlStem); QUrlQuery q(url); const QString title = query[queryKeyTitle]; q.addQueryItem(QStringLiteral("ti"), title); const QString authors = query[queryKeyAuthor]; q.addQueryItem(QStringLiteral("au"), authors); const QString year = query[queryKeyYear]; if (!year.isEmpty()) q.addQueryItem(QStringLiteral("year"), year); q.addQueryItem(QStringLiteral("format"), QStringLiteral("bibtex")); url.setQuery(q); QNetworkRequest request(url); QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchMRLookup::doneFetchingResultPage); refreshBusyProperty(); } QString OnlineSearchMRLookup::label() const { return i18n("MR Lookup"); } QString OnlineSearchMRLookup::favIconUrl() const { return QStringLiteral("http://www.ams.org/favicon.ico"); } QUrl OnlineSearchMRLookup::homepage() const { return QUrl(QStringLiteral("https://mathscinet.ams.org/mrlookup")); } void OnlineSearchMRLookup::doneFetchingResultPage() { QNetworkReply *reply = static_cast(sender()); emit progress(curStep = numSteps, numSteps); if (handleErrors(reply)) { /// ensure proper treatment of UTF-8 characters QString htmlCode = QString::fromUtf8(reply->readAll().constData()); QString bibtexCode; int p1 = -1, p2 = -1; while ((p1 = htmlCode.indexOf(QStringLiteral("
"), p2 + 1)) >= 0 && (p2 = htmlCode.indexOf(QStringLiteral("
"), p1 + 1)) >= 0) { bibtexCode += htmlCode.midRef(p1 + 5, p2 - p1 - 5); bibtexCode += QLatin1Char('\n'); } FileImporterBibTeX importer(this); File *bibtexFile = importer.fromString(bibtexCode); bool hasEntry = false; if (bibtexFile != nullptr) { for (const auto &element : const_cast(*bibtexFile)) { const QSharedPointer entry = element.dynamicCast(); hasEntry |= publishEntry(entry); } delete bibtexFile; } stopSearch(hasEntry ? resultNoError : resultUnspecifiedError); } refreshBusyProperty(); } void OnlineSearchMRLookup::sanitizeEntry(QSharedPointer entry) { OnlineSearchAbstract::sanitizeEntry(entry); /// Rewrite 'fjournal' fields to become 'journal' fields /// (overwriting them if necessary) const QString ftFJournal = QStringLiteral("fjournal"); if (entry->contains(ftFJournal)) { Value v = entry->value(ftFJournal); entry->remove(Entry::ftJournal); entry->remove(ftFJournal); entry->insert(Entry::ftJournal, v); } /// Remove URL from entry if contains a DOI and the DOI field is present if (entry->contains(Entry::ftDOI) && entry->contains(Entry::ftUrl)) { Value v = entry->value(Entry::ftUrl); if (v.containsPattern(QStringLiteral("http://dx.doi.org"))) { entry->remove(Entry::ftUrl); } } } diff --git a/src/networking/onlinesearch/onlinesearchmrlookup.h b/src/networking/onlinesearch/onlinesearchmrlookup.h index 256b6901..6012080e 100644 --- a/src/networking/onlinesearch/onlinesearchmrlookup.h +++ b/src/networking/onlinesearch/onlinesearchmrlookup.h @@ -1,54 +1,54 @@ /*************************************************************************** * Copyright (C) 2004-2019 by Thomas Fischer * * 2014 Pavel Zorin-Kranich * * 2018 Alexander Dunlap * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #ifndef KBIBTEX_NETWORKING_ONLINESEARCHMRLOOKUP_H #define KBIBTEX_NETWORKING_ONLINESEARCHMRLOOKUP_H #include #ifdef HAVE_KF5 #include "kbibtexnetworking_export.h" #endif // HAVE_KF5 /** * @author Pavel Zorin-Kranich */ class KBIBTEXNETWORKING_EXPORT OnlineSearchMRLookup : public OnlineSearchAbstract { Q_OBJECT public: - explicit OnlineSearchMRLookup(QWidget *parent); + explicit OnlineSearchMRLookup(QObject *parent); void startSearch(const QMap &query, int numResults) override; QString label() const override; QUrl homepage() const override; protected: QString favIconUrl() const override; void sanitizeEntry(QSharedPointer entry) override; private slots: void doneFetchingResultPage(); private: static const QString queryUrlStem; }; #endif // KBIBTEX_NETWORKING_ONLINESEARCHMRLOOKUP_H diff --git a/src/networking/onlinesearch/onlinesearchsimplebibtexdownload.cpp b/src/networking/onlinesearch/onlinesearchsimplebibtexdownload.cpp index dfd4cb88..9dccce70 100644 --- a/src/networking/onlinesearch/onlinesearchsimplebibtexdownload.cpp +++ b/src/networking/onlinesearch/onlinesearchsimplebibtexdownload.cpp @@ -1,112 +1,112 @@ /*************************************************************************** * Copyright (C) 2004-2018 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #include "onlinesearchsimplebibtexdownload.h" #include #include #include #include #include "internalnetworkaccessmanager.h" #include "logging_networking.h" -OnlineSearchSimpleBibTeXDownload::OnlineSearchSimpleBibTeXDownload(QWidget *parent) +OnlineSearchSimpleBibTeXDownload::OnlineSearchSimpleBibTeXDownload(QObject *parent) : OnlineSearchAbstract(parent) { /// nothing } void OnlineSearchSimpleBibTeXDownload::startSearch(const QMap &query, int numResults) { m_hasBeenCanceled = false; emit progress(curStep = 0, numSteps = 2); QNetworkRequest request(buildQueryUrl(query, numResults)); QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); InternalNetworkAccessManager::instance().setNetworkReplyTimeout(reply); connect(reply, &QNetworkReply::finished, this, &OnlineSearchSimpleBibTeXDownload::downloadDone); refreshBusyProperty(); } QString OnlineSearchSimpleBibTeXDownload::processRawDownload(const QString &download) { /// Default implementation does not do anything return download; } void OnlineSearchSimpleBibTeXDownload::downloadDone() { emit progress(++curStep, numSteps = 2); QNetworkReply *reply = static_cast(sender()); if (handleErrors(reply)) { /// ensure proper treatment of UTF-8 characters QString bibTeXcode = QString::fromUtf8(reply->readAll().constData()); if (bibTeXcode.contains(QStringLiteral("]*[/]?>")); bibTeXcode = bibTeXcode.remove(htmlLinebreakRegExp); /// Find first BibTeX entry in HTML code, clip away all HTML code before that static const QRegularExpression elementTypeRegExp(QStringLiteral("[@]\\S+\\{")); int p1 = -1; QRegularExpressionMatchIterator elementTypeRegExpMatchIt = elementTypeRegExp.globalMatch(bibTeXcode); while (p1 < 0 && elementTypeRegExpMatchIt.hasNext()) { const QRegularExpressionMatch elementTypeRegExpMatch = elementTypeRegExpMatchIt.next(); /// Hop over JavaScript's "@import" statements if (elementTypeRegExpMatch.captured(0) != QStringLiteral("@import{")) p1 = elementTypeRegExpMatch.capturedStart(); } if (p1 > 1) bibTeXcode = bibTeXcode.mid(p1); /// Find HTML code after BibTeX code, clip that away, too static const QRegularExpression htmlContinuationRegExp(QStringLiteral("<[/]?\\S+")); p1 = bibTeXcode.indexOf(htmlContinuationRegExp); if (p1 > 1) bibTeXcode = bibTeXcode.left(p1 - 1); } if (!bibTeXcode.isEmpty()) { FileImporterBibTeX importer(this); File *bibtexFile = importer.fromString(bibTeXcode); bool hasEntries = false; if (bibtexFile != nullptr) { for (const auto &element : const_cast(*bibtexFile)) { QSharedPointer entry = element.dynamicCast(); hasEntries |= publishEntry(entry); } stopSearch(resultNoError); delete bibtexFile; } else { qCWarning(LOG_KBIBTEX_NETWORKING) << "No valid BibTeX file results returned on request on" << InternalNetworkAccessManager::removeApiKey(reply->url()).toDisplayString(); stopSearch(resultUnspecifiedError); } } else { /// returned file is empty stopSearch(resultNoError); } } refreshBusyProperty(); } diff --git a/src/networking/onlinesearch/onlinesearchsimplebibtexdownload.h b/src/networking/onlinesearch/onlinesearchsimplebibtexdownload.h index 46b4d057..e8b78e28 100644 --- a/src/networking/onlinesearch/onlinesearchsimplebibtexdownload.h +++ b/src/networking/onlinesearch/onlinesearchsimplebibtexdownload.h @@ -1,47 +1,47 @@ /*************************************************************************** * Copyright (C) 2004-2019 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #ifndef KBIBTEX_NETWORKING_ONLINESEARCHSIMPLEBIBTEXDOWNLOAD_H #define KBIBTEX_NETWORKING_ONLINESEARCHSIMPLEBIBTEXDOWNLOAD_H #include #ifdef HAVE_KF5 #include "kbibtexnetworking_export.h" #endif // HAVE_KF5 /** * @author Thomas Fischer */ class KBIBTEXNETWORKING_EXPORT OnlineSearchSimpleBibTeXDownload : public OnlineSearchAbstract { Q_OBJECT public: - explicit OnlineSearchSimpleBibTeXDownload(QWidget *parent); + explicit OnlineSearchSimpleBibTeXDownload(QObject *parent); void startSearch(const QMap &query, int numResults) override; protected: virtual QUrl buildQueryUrl(const QMap &query, int numResults) = 0; virtual QString processRawDownload(const QString &download); private slots: void downloadDone(); }; #endif // KBIBTEX_NETWORKING_ONLINESEARCHSIMPLEBIBTEXDOWNLOAD_H diff --git a/src/networking/onlinesearch/onlinesearchsoanasaads.cpp b/src/networking/onlinesearch/onlinesearchsoanasaads.cpp index 47ab7996..c01689ce 100644 --- a/src/networking/onlinesearch/onlinesearchsoanasaads.cpp +++ b/src/networking/onlinesearch/onlinesearchsoanasaads.cpp @@ -1,96 +1,96 @@ /*************************************************************************** * Copyright (C) 2004-2018 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #include "onlinesearchsoanasaads.h" #include -OnlineSearchSOANASAADS::OnlineSearchSOANASAADS(QWidget *parent) +OnlineSearchSOANASAADS::OnlineSearchSOANASAADS(QObject *parent) : OnlineSearchSimpleBibTeXDownload(parent) { /// nothing } QString OnlineSearchSOANASAADS::label() const { return i18n("SAO/NASA Astrophysics Data System (ADS)"); } QUrl OnlineSearchSOANASAADS::homepage() const { return QUrl(QStringLiteral("http://adswww.harvard.edu/")); } QString OnlineSearchSOANASAADS::favIconUrl() const { return QStringLiteral("http://adsabs.harvard.edu/favicon.ico"); } QUrl OnlineSearchSOANASAADS::buildQueryUrl(const QMap &query, int numResults) { static const QString globalSearch = QStringLiteral("\"%1\""); static const QString rangeSearch = QStringLiteral("%1:\"%2\""); // TODO /// http://adsabs.harvard.edu/cgi-bin/basic_connect?qsearch=Hansen&version=1&data_type=BIBTEXPLUS&type=FILE&sort=NDATE&nr_to_return=5 const QStringList freeTextWords = splitRespectingQuotationMarks(query[queryKeyFreeText]); const QStringList yearWords = splitRespectingQuotationMarks(query[queryKeyYear]); const QStringList titleWords = splitRespectingQuotationMarks(query[queryKeyTitle]); const QStringList authorWords = splitRespectingQuotationMarks(query[queryKeyAuthor]); /// append search terms QStringList queryFragments; queryFragments.reserve(freeTextWords.size() + yearWords.size() + titleWords.size() + authorWords.size()); /// add words from "free text" field for (const QString &word : freeTextWords) queryFragments.append(globalSearch.arg(word)); /// add words from "year" field for (const QString &word : yearWords) queryFragments.append(word); ///< no quotation marks around years /// add words from "title" field for (const QString &word : titleWords) queryFragments.append(rangeSearch.arg(QStringLiteral("intitle"), word)); /// add words from "author" field for (const QString &word : authorWords) queryFragments.append(rangeSearch.arg(QStringLiteral("author"), word)); /// Build URL QString urlText = QStringLiteral("http://adsabs.harvard.edu/cgi-bin/basic_connect?version=1&data_type=BIBTEXPLUS&type=FILE&sort=NDATE&qsearch="); urlText.append(queryFragments.join(QStringLiteral("+")).replace(QLatin1Char('"'), QStringLiteral("%22"))); /// set number of expected results urlText.append(QString(QStringLiteral("&nr_to_return=%1")).arg(numResults)); return QUrl(urlText); } QString OnlineSearchSOANASAADS::processRawDownload(const QString &download) { /// Skip HTML header and some other non-BibTeX content const int p1 = download.indexOf(QStringLiteral("abstracts, starting")); if (p1 > 1) { const int p2 = download.indexOf(QStringLiteral("\n"), p1); if (p2 > p1) return download.mid(p2 + 1); } /// Skipping header failed, return input text as it was return download; } diff --git a/src/networking/onlinesearch/onlinesearchsoanasaads.h b/src/networking/onlinesearch/onlinesearchsoanasaads.h index ac7330c1..45c3e039 100644 --- a/src/networking/onlinesearch/onlinesearchsoanasaads.h +++ b/src/networking/onlinesearch/onlinesearchsoanasaads.h @@ -1,46 +1,46 @@ /*************************************************************************** * Copyright (C) 2004-2019 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #ifndef KBIBTEX_NETWORKING_ONLINESEARCHSOANASAADS_H #define KBIBTEX_NETWORKING_ONLINESEARCHSOANASAADS_H #include #ifdef HAVE_KF5 #include "kbibtexnetworking_export.h" #endif // HAVE_KF5 /** * @author Thomas Fischer */ class KBIBTEXNETWORKING_EXPORT OnlineSearchSOANASAADS : public OnlineSearchSimpleBibTeXDownload { Q_OBJECT public: - explicit OnlineSearchSOANASAADS(QWidget *parent); + explicit OnlineSearchSOANASAADS(QObject *parent); QString label() const override; QUrl homepage() const override; protected: QString favIconUrl() const override; QUrl buildQueryUrl(const QMap &query, int numResults) override; QString processRawDownload(const QString &download) override; }; #endif // KBIBTEX_NETWORKING_ONLINESEARCHSOANASAADS_H