diff --git a/src/fetch/amazonfetcher.cpp b/src/fetch/amazonfetcher.cpp index 2fb09ba1..a7e5acfd 100644 --- a/src/fetch/amazonfetcher.cpp +++ b/src/fetch/amazonfetcher.cpp @@ -1,1117 +1,1119 @@ /*************************************************************************** Copyright (C) 2004-2020 Robby Stephenson ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * * * ***************************************************************************/ #include #include "amazonfetcher.h" #include "amazonrequest.h" #include "../collectionfactory.h" #include "../images/imagefactory.h" #include "../utils/guiproxy.h" #include "../collection.h" #include "../entry.h" #include "../field.h" #include "../fieldformat.h" #include "../utils/string_utils.h" #include "../utils/isbnvalidator.h" #include "../gui/combobox.h" #include "../tellico_debug.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace { static const int AMAZON_RETURNS_PER_REQUEST = 10; static const int AMAZON_MAX_RETURNS_TOTAL = 20; static const char* AMAZON_ASSOC_TOKEN = "tellico-20"; } using namespace Tellico; using Tellico::Fetch::AmazonFetcher; // static // see https://webservices.amazon.com/paapi5/documentation/common-request-parameters.html#host-and-region const AmazonFetcher::SiteData& AmazonFetcher::siteData(int site_) { Q_ASSERT(site_ >= 0); Q_ASSERT(site_ < XX); static SiteData dataVector[16] = { { i18n("Amazon (US)"), "www.amazon.com", "us-east-1", QLatin1String("us"), i18n("United States") }, { i18n("Amazon (UK)"), "www.amazon.co.uk", "eu-west-1", QLatin1String("gb"), i18n("United Kingdom") }, { i18n("Amazon (Germany)"), "www.amazon.de", "eu-west-1", QLatin1String("de"), i18n("Germany") }, { i18n("Amazon (Japan)"), "www.amazon.co.jp", "us-west-2", QLatin1String("jp"), i18n("Japan") }, { i18n("Amazon (France)"), "www.amazon.fr", "eu-west-1", QLatin1String("fr"), i18n("France") }, { i18n("Amazon (Canada)"), "www.amazon.ca", "us-east-1", QLatin1String("ca"), i18n("Canada") }, { // TODO: no chinese in PAAPI-5 yet? i18n("Amazon (China)"), "http://webservices.amazon.cn/onca/xml", "us-west-2", QLatin1String("ch"), i18n("China") }, { i18n("Amazon (Spain)"), "www.amazon.es", "eu-west-1", QLatin1String("es"), i18n("Spain") }, { i18n("Amazon (Italy)"), "www.amazon.it", "eu-west-1", QLatin1String("it"), i18n("Italy") }, { i18n("Amazon (Brazil)"), "www.amazon.com.br", "us-east-1", QLatin1String("br"), i18n("Brazil") }, { i18n("Amazon (Australia)"), "www.amazon.com.au", "us-west-2", QLatin1String("au"), i18n("Australia") }, { i18n("Amazon (India)"), "www.amazon.in", "eu-west-1", QLatin1String("in"), i18n("India") }, { i18n("Amazon (Mexico)"), "www.amazon.com.mx", "us-east-1", QLatin1String("mx"), i18n("Mexico") }, { i18n("Amazon (Turkey)"), "www.amazon.com.tr", "eu-west-1", QLatin1String("tr"), i18n("Turkey") }, { i18n("Amazon (Singapore)"), "www.amazon.sg", "us-west-2", QLatin1String("sg"), i18n("Singapore") }, { i18n("Amazon (UAE)"), "www.amazon.ae", "eu-west-1", QLatin1String("ae"), i18n("United Arab Emirates") } }; return dataVector[qBound(0, site_, static_cast(sizeof(dataVector)/sizeof(SiteData)))]; } AmazonFetcher::AmazonFetcher(QObject* parent_) : Fetcher(parent_), m_site(Unknown), m_imageSize(MediumImage), m_assoc(QLatin1String(AMAZON_ASSOC_TOKEN)), m_limit(AMAZON_MAX_RETURNS_TOTAL), m_countOffset(0), m_page(1), m_total(-1), m_numResults(0), m_job(nullptr), m_started(false) { } AmazonFetcher::~AmazonFetcher() { } QString AmazonFetcher::source() const { return m_name.isEmpty() ? defaultName() : m_name; } QString AmazonFetcher::attribution() const { return i18n("This data is licensed under specific terms.", QLatin1String("https://affiliate-program.amazon.com/gp/advertising/api/detail/agreement.html")); } bool AmazonFetcher::canFetch(int type) const { return type == Data::Collection::Book || type == Data::Collection::ComicBook || type == Data::Collection::Bibtex || type == Data::Collection::Album || type == Data::Collection::Video || type == Data::Collection::Game || type == Data::Collection::BoardGame; } bool AmazonFetcher::canSearch(FetchKey k) const { // no UPC in Canada return k == Title || k == Person || k == ISBN || k == UPC || k == Keyword; } void AmazonFetcher::readConfigHook(const KConfigGroup& config_) { const int site = config_.readEntry("Site", int(Unknown)); Q_ASSERT(site != Unknown); m_site = static_cast(site); if(m_name.isEmpty()) { m_name = siteData(m_site).title; } QString s = config_.readEntry("AccessKey"); if(!s.isEmpty()) { m_accessKey = s; } else { myWarning() << "No Amazon access key"; } s = config_.readEntry("AssocToken"); if(!s.isEmpty()) { m_assoc = s; } s = config_.readEntry("SecretKey"); if(!s.isEmpty()) { m_secretKey = s; } else { myWarning() << "No Amazon secret key"; } int imageSize = config_.readEntry("Image Size", -1); if(imageSize > -1) { m_imageSize = static_cast(imageSize); } } void AmazonFetcher::search() { m_started = true; m_page = 1; m_total = -1; m_countOffset = 0; m_numResults = 0; doSearch(); } void AmazonFetcher::continueSearch() { m_started = true; m_limit += AMAZON_MAX_RETURNS_TOTAL; doSearch(); } void AmazonFetcher::doSearch() { if(m_secretKey.isEmpty() || m_accessKey.isEmpty()) { // this message is split in two since the first half is reused later message(i18n("Access to data from Amazon.com requires an AWS Access Key ID and a Secret Key.") + QLatin1Char(' ') + i18n("Those values must be entered in the data source settings."), MessageHandler::Error); stop(); return; } const QByteArray payload = requestPayload(request()); if(payload.isEmpty()) { stop(); return; } AmazonRequest request(m_accessKey, m_secretKey); request.setHost(siteData(m_site).host); request.setRegion(siteData(m_site).region); // debugging check if(m_testResultsFile.isEmpty()) { QUrl u; u.setHost(QString::fromUtf8(siteData(m_site).host)); m_job = KIO::storedHttpPost(payload, u, KIO::HideProgressInfo); QMapIterator i(request.headers(payload)); while(i.hasNext()) { i.next(); m_job->addMetaData(QStringLiteral("customHTTPHeader"), QString::fromUtf8(i.key() + ": " + i.value())); } } else { myDebug() << "Reading" << m_testResultsFile; m_job = KIO::storedGet(QUrl::fromLocalFile(m_testResultsFile), KIO::NoReload, KIO::HideProgressInfo); } KJobWidgets::setWindow(m_job, GUI::Proxy::widget()); connect(m_job.data(), &KJob::result, this, &AmazonFetcher::slotComplete); } void AmazonFetcher::stop() { if(!m_started) { return; } // myDebug(); if(m_job) { m_job->kill(); m_job = nullptr; } m_started = false; emit signalDone(this); } void AmazonFetcher::slotComplete(KJob*) { if(m_job->error()) { m_job->uiDelegate()->showErrorMessage(); stop(); return; } const QByteArray data = m_job->data(); if(data.isEmpty()) { myDebug() << "no data"; stop(); return; } // since the fetch is done, don't worry about holding the job pointer m_job = nullptr; #if 0 myWarning() << "Remove debug from amazonfetcher.cpp"; QFile f(QString::fromLatin1("/tmp/test%1.xml").arg(m_page)); if(f.open(QIODevice::WriteOnly)) { QTextStream t(&f); t.setCodec("UTF-8"); t << data; } f.close(); #endif QJsonParseError jsonError; QJsonObject databject = QJsonDocument::fromJson(data, &jsonError).object(); if(jsonError.error != QJsonParseError::NoError) { myDebug() << "AmazonFetcher: JSON error -" << jsonError.errorString(); message(jsonError.errorString(), MessageHandler::Error); stop(); return; } QJsonObject resultObject = databject.value(QStringLiteral("SearchResult")).toObject(); if(resultObject.isEmpty()) { resultObject = databject.value(QStringLiteral("ItemsResult")).toObject(); } if(m_total == -1) { int totalResults = resultObject.value(QStringLiteral("TotalResultCount")).toInt(); if(totalResults > 0) { m_total = totalResults; // myDebug() << "Total results is" << totalResults; } } QStringList errors; QJsonValue errorValue = databject.value(QLatin1String("Errors")); if(!errorValue.isNull()) { foreach(const QJsonValue& error, errorValue.toArray()) { errors += error.toObject().value(QLatin1String("Message")).toString(); } } if(!errors.isEmpty()) { for(QStringList::ConstIterator it = errors.constBegin(); it != errors.constEnd(); ++it) { myDebug() << "AmazonFetcher::" << *it; } message(errors[0], MessageHandler::Error); stop(); return; } Data::CollPtr coll = createCollection(); if(!coll) { myDebug() << "no collection pointer"; stop(); return; } int count = -1; foreach(const QJsonValue& item, resultObject.value(QLatin1String("Items")).toArray()) { ++count; if(m_numResults >= m_limit) { break; } if(!m_started) { // might get aborted break; } Data::EntryPtr entry(new Data::Entry(coll)); populateEntry(entry, item.toObject()); // special case book author // amazon is really bad about not putting spaces after periods if(coll->type() == Data::Collection::Book) { QRegExp rx(QLatin1String("\\.([^\\s])")); QStringList values = FieldFormat::splitValue(entry->field(QStringLiteral("author"))); for(QStringList::Iterator it = values.begin(); it != values.end(); ++it) { (*it).replace(rx, QStringLiteral(". \\1")); } entry->setField(QStringLiteral("author"), values.join(FieldFormat::delimiterString())); } // UK puts the year in the title for some reason if(m_site == UK && coll->type() == Data::Collection::Video) { QRegExp rx(QLatin1String("\\[(\\d{4})\\]")); QString t = entry->title(); if(rx.indexIn(t) > -1) { QString y = rx.cap(1); t = t.remove(rx).simplified(); entry->setField(QStringLiteral("title"), t); if(entry->field(QStringLiteral("year")).isEmpty()) { entry->setField(QStringLiteral("year"), y); } } } // strip HTML from comments, or plot in movies // tentatively don't do this, looks like ECS 4 cleaned everything up /* if(coll->type() == Data::Collection::Video) { QString plot = entry->field(QLatin1String("plot")); plot.remove(stripHTML); entry->setField(QLatin1String("plot"), plot); } else if(coll->type() == Data::Collection::Game) { QString desc = entry->field(QLatin1String("description")); desc.remove(stripHTML); entry->setField(QLatin1String("description"), desc); } else { QString comments = entry->field(QLatin1String("comments")); comments.remove(stripHTML); entry->setField(QLatin1String("comments"), comments); } */ // myDebug() << entry->title(); FetchResult* r = new FetchResult(Fetcher::Ptr(this), entry); m_entries.insert(r->uid, entry); emit signalResultFound(r); ++m_numResults; } // we might have gotten aborted if(!m_started) { return; } // are there any additional results to get? m_hasMoreResults = m_page * AMAZON_RETURNS_PER_REQUEST < m_total; const int currentTotal = qMin(m_total, m_limit); if(m_page * AMAZON_RETURNS_PER_REQUEST < currentTotal) { int foundCount = (m_page-1) * AMAZON_RETURNS_PER_REQUEST + coll->entryCount(); message(i18n("Results from %1: %2/%3", source(), foundCount, m_total), MessageHandler::Status); ++m_page; m_countOffset = 0; doSearch(); } else if(request().value.count(QLatin1Char(';')) > 9) { // start new request after cutting off first 10 isbn values FetchRequest newRequest = request(); newRequest.value = request().value.section(QLatin1Char(';'), 10); startSearch(newRequest); } else { m_countOffset = m_entries.count() % AMAZON_RETURNS_PER_REQUEST; if(m_countOffset == 0) { ++m_page; // need to go to next page } stop(); } } Tellico::Data::EntryPtr AmazonFetcher::fetchEntryHook(uint uid_) { Data::EntryPtr entry = m_entries[uid_]; if(!entry) { myWarning() << "no entry in dict"; return entry; } // do what we can to remove useless keywords const int type = collectionType(); switch(type) { case Data::Collection::Book: case Data::Collection::ComicBook: case Data::Collection::Bibtex: if(optionalFields().contains(QStringLiteral("keyword"))) { StringSet newWords; const QStringList keywords = FieldFormat::splitValue(entry->field(QStringLiteral("keyword"))); foreach(const QString& keyword, keywords) { if(keyword == QLatin1String("General") || keyword == QLatin1String("Subjects") || keyword == QLatin1String("Par prix") || // french stuff keyword == QLatin1String("Divers") || // french stuff keyword.startsWith(QLatin1Char('(')) || keyword.startsWith(QLatin1String("Authors"))) { continue; } newWords.add(keyword); } entry->setField(QStringLiteral("keyword"), newWords.toList().join(FieldFormat::delimiterString())); } entry->setField(QStringLiteral("comments"), Tellico::decodeHTML(entry->field(QStringLiteral("comments")))); break; case Data::Collection::Video: { const QString genres = QStringLiteral("genre"); QStringList oldWords = FieldFormat::splitValue(entry->field(genres)); StringSet words; // only care about genres that have "Genres" in the amazon response // and take the first word after that for(QStringList::Iterator it = oldWords.begin(); it != oldWords.end(); ++it) { if((*it).indexOf(QLatin1String("Genres")) == -1) { continue; } // the amazon2tellico stylesheet separates words with '/' QStringList nodes = (*it).split(QLatin1Char('/')); for(QStringList::Iterator it2 = nodes.begin(); it2 != nodes.end(); ++it2) { if(*it2 != QLatin1String("Genres")) { continue; } ++it2; if(it2 != nodes.end() && *it2 != QLatin1String("General")) { words.add(*it2); } break; // we're done } } entry->setField(genres, words.toList().join(FieldFormat::delimiterString())); // language tracks get duplicated, too words.clear(); words.add(FieldFormat::splitValue(entry->field(QStringLiteral("language")))); entry->setField(QStringLiteral("language"), words.toList().join(FieldFormat::delimiterString())); } entry->setField(QStringLiteral("plot"), Tellico::decodeHTML(entry->field(QStringLiteral("plot")))); break; case Data::Collection::Album: { const QString genres = QStringLiteral("genre"); QStringList oldWords = FieldFormat::splitValue(entry->field(genres)); StringSet words; // only care about genres that have "Styles" in the amazon response // and take the first word after that for(QStringList::Iterator it = oldWords.begin(); it != oldWords.end(); ++it) { if((*it).indexOf(QLatin1String("Styles")) == -1) { continue; } // the amazon2tellico stylesheet separates words with '/' QStringList nodes = (*it).split(QLatin1Char('/')); bool isStyle = false; for(QStringList::Iterator it2 = nodes.begin(); it2 != nodes.end(); ++it2) { if(!isStyle) { if(*it2 == QLatin1String("Styles")) { isStyle = true; } continue; } if(*it2 != QLatin1String("General")) { words.add(*it2); } } } entry->setField(genres, words.toList().join(FieldFormat::delimiterString())); } entry->setField(QStringLiteral("comments"), Tellico::decodeHTML(entry->field(QStringLiteral("comments")))); break; case Data::Collection::Game: entry->setField(QStringLiteral("description"), Tellico::decodeHTML(entry->field(QStringLiteral("description")))); break; } // clean up the title parseTitle(entry); // also sometimes table fields have rows but no values Data::FieldList fields = entry->collection()->fields(); QRegExp blank(QLatin1String("[\\s") + FieldFormat::columnDelimiterString() + FieldFormat::delimiterString() + QLatin1String("]+")); // only white space, column separators and value separators foreach(Data::FieldPtr fIt, fields) { if(fIt->type() != Data::Field::Table) { continue; } if(blank.exactMatch(entry->field(fIt))) { entry->setField(fIt, QString()); } } // don't want to show image urls in the fetch dialog // so clear them after reading the URL QString imageURL; switch(m_imageSize) { case SmallImage: imageURL = entry->field(QStringLiteral("small-image")); entry->setField(QStringLiteral("small-image"), QString()); break; case MediumImage: imageURL = entry->field(QStringLiteral("medium-image")); entry->setField(QStringLiteral("medium-image"), QString()); break; case LargeImage: imageURL = entry->field(QStringLiteral("large-image")); entry->setField(QStringLiteral("large-image"), QString()); break; case NoImage: default: break; } // myDebug() << "grabbing " << imageURL; if(!imageURL.isEmpty()) { QString id = ImageFactory::addImage(QUrl::fromUserInput(imageURL), true); if(id.isEmpty()) { message(i18n("The cover image could not be loaded."), MessageHandler::Warning); } else { // amazon serves up 1x1 gifs occasionally, but that's caught in the image constructor // all relevant collection types have cover fields entry->setField(QStringLiteral("cover"), id); } } return entry; } Tellico::Fetch::FetchRequest AmazonFetcher::updateRequest(Data::EntryPtr entry_) { const int type = entry_->collection()->type(); const QString t = entry_->field(QStringLiteral("title")); if(type == Data::Collection::Book || type == Data::Collection::ComicBook || type == Data::Collection::Bibtex) { const QString isbn = entry_->field(QStringLiteral("isbn")); if(!isbn.isEmpty()) { return FetchRequest(Fetch::ISBN, isbn); } const QString a = entry_->field(QStringLiteral("author")); if(!a.isEmpty()) { return t.isEmpty() ? FetchRequest(Fetch::Person, a) : FetchRequest(Fetch::Keyword, t + QLatin1Char('-') + a); } } else if(type == Data::Collection::Album) { const QString a = entry_->field(QStringLiteral("artist")); if(!a.isEmpty()) { return t.isEmpty() ? FetchRequest(Fetch::Person, a) : FetchRequest(Fetch::Keyword, t + QLatin1Char('-') + a); } } // optimistically try searching for title and rely on Collection::sameEntry() to figure things out if(!t.isEmpty()) { return FetchRequest(Fetch::Title, t); } return FetchRequest(); } QByteArray AmazonFetcher::requestPayload(FetchRequest request_) { QJsonObject payload; payload.insert(QLatin1String("PartnerTag"), m_assoc); payload.insert(QLatin1String("PartnerType"), QLatin1String("Associates")); payload.insert(QLatin1String("Operation"), QLatin1String("SearchItems")); payload.insert(QLatin1String("SortBy"), QLatin1String("Relevance")); // not mandatory // payload.insert(QLatin1String("Marketplace"), QLatin1String(siteData(m_site).host)); if(m_page > 1) { payload.insert(QLatin1String("ItemPage"), m_page); } QJsonArray resources; resources.append(QLatin1String("ItemInfo.Title")); const int type = request_.collectionType; switch(type) { case Data::Collection::Book: case Data::Collection::ComicBook: case Data::Collection::Bibtex: payload.insert(QLatin1String("SearchIndex"), QLatin1String("Books")); resources.append(QLatin1String("ItemInfo.ExternalIds")); break; case Data::Collection::Album: payload.insert(QLatin1String("SearchIndex"), QLatin1String("Music")); break; case Data::Collection::Video: // CA and JP appear to have a bug where Video only returns VHS or Music results // DVD will return DVD, Blu-ray, etc. so just ignore VHS for those users payload.insert(QLatin1String("SearchIndex"), QLatin1String("MoviesAndTV")); if(m_site == CA || m_site == JP || m_site == IT || m_site == ES) { payload.insert(QStringLiteral("SearchIndex"), QStringLiteral("DVD")); } else { payload.insert(QStringLiteral("SearchIndex"), QStringLiteral("Video")); } // params.insert(QStringLiteral("SortIndex"), QStringLiteral("relevancerank")); break; case Data::Collection::Game: payload.insert(QLatin1String("SearchIndex"), QLatin1String("VideoGames")); break; case Data::Collection::BoardGame: payload.insert(QLatin1String("SearchIndex"), QLatin1String("ToysAndGames")); // params.insert(QStringLiteral("SortIndex"), QStringLiteral("relevancerank")); break; case Data::Collection::Coin: case Data::Collection::Stamp: case Data::Collection::Wine: case Data::Collection::Base: case Data::Collection::Card: myDebug() << "can't fetch this type:" << collectionType(); return QByteArray(); } switch(request_.key) { case Title: payload.insert(QLatin1String("Title"), request_.value); break; case Person: if(type == Data::Collection::Video) { payload.insert(QStringLiteral("Actor"), request_.value); // payload.insert(QStringLiteral("Director"), request_.value); } else if(type == Data::Collection::Album) { payload.insert(QStringLiteral("Artist"), request_.value); } else if(type == Data::Collection::Book) { payload.insert(QLatin1String("Author"), request_.value); } else { payload.insert(QLatin1String("Keywords"), request_.value); } break; case ISBN: { QString cleanValue = request_.value; cleanValue.remove(QLatin1Char('-')); // ISBN only get digits or 'X' QStringList isbns = FieldFormat::splitValue(cleanValue); // Amazon isbn13 search is still very flaky, so if possible, we're going to convert // all of them to isbn10. If we run into a 979 isbn13, then we're forced to do an // isbn13 search bool isbn13 = false; for(QStringList::Iterator it = isbns.begin(); it != isbns.end(); ) { if((*it).startsWith(QLatin1String("979"))) { isbn13 = true; break; } ++it; } // if we want isbn10, then convert all if(!isbn13) { for(QStringList::Iterator it = isbns.begin(); it != isbns.end(); ++it) { if((*it).length() > 12) { (*it) = ISBNValidator::isbn10(*it); (*it).remove(QLatin1Char('-')); } } } // limit to first 10 while(isbns.size() > 10) { isbns.pop_back(); } payload.insert(QLatin1String("Keywords"), isbns.join(QLatin1String("|"))); if(isbn13) { // params.insert(QStringLiteral("IdType"), QStringLiteral("EAN")); } } break; case UPC: { QString cleanValue = request_.value; cleanValue.remove(QLatin1Char('-')); // for EAN values, add 0 to beginning if not 13 characters // in order to assume US country code from UPC value QStringList values; foreach(const QString& splitValue, cleanValue.split(FieldFormat::delimiterString())) { QString tmpValue = splitValue; if(m_site != US && tmpValue.length() == 12) { tmpValue.prepend(QLatin1Char('0')); } values << tmpValue; // limit to first 10 values if(values.length() >= 10) { break; } } payload.insert(QLatin1String("Keywords"), values.join(QLatin1String("|"))); } break; case Keyword: payload.insert(QLatin1String("Keywords"), request_.value); break; case Raw: { QString key = request_.value.section(QLatin1Char('='), 0, 0).trimmed(); QString str = request_.value.section(QLatin1Char('='), 1).trimmed(); payload.insert(key, str); } break; default: myWarning() << "key not recognized: " << request().key; return QByteArray(); } switch(m_imageSize) { case SmallImage: resources.append(QLatin1String("Images.Primary.Small")); break; case MediumImage: resources.append(QLatin1String("Images.Primary.Medium")); break; case LargeImage: resources.append(QLatin1String("Images.Primary.Large")); break; case NoImage: break; } payload.insert(QLatin1String("Resources"), resources); return QJsonDocument(payload).toJson(QJsonDocument::Compact); } Tellico::Data::CollPtr AmazonFetcher::createCollection() { Data::CollPtr coll = CollectionFactory::collection(collectionType(), true); if(!coll) { return coll; } QString imageFieldName; switch(m_imageSize) { case SmallImage: imageFieldName = QStringLiteral("small-image"); break; case MediumImage: imageFieldName = QStringLiteral("medium-image"); break; case LargeImage: imageFieldName = QStringLiteral("large-image"); break; case NoImage: break; } if(!imageFieldName.isEmpty()) { Data::FieldPtr field(new Data::Field(imageFieldName, QString(), Data::Field::URL)); coll->addField(field); } if(optionalFields().contains(QStringLiteral("amazon"))) { Data::FieldPtr field(new Data::Field(QStringLiteral("amazon"), i18n("Amazon Link"), Data::Field::URL)); field->setCategory(i18n("General")); coll->addField(field); } return coll; } void AmazonFetcher::populateEntry(Data::EntryPtr entry_, const QJsonObject& info_) { QVariantMap itemMap = info_.value(QLatin1String("ItemInfo")).toObject().toVariantMap(); entry_->setField(QStringLiteral("title"), mapValue(itemMap, "Title", "DisplayValue")); const QString isbn = mapValue(itemMap, "ExternalIds", "ISBNs", "DisplayValues"); if(!isbn.isEmpty()) { entry_->setField(QStringLiteral("isbn"), isbn); } QVariantMap contentMap = itemMap.value(QLatin1String("ContentInfo")).toMap(); entry_->setField(QStringLiteral("pages"), mapValue(contentMap, "PagesCount", "DisplayValue")); const QString pubDate = mapValue(contentMap, "PublicationDate", "DisplayValue"); if(!pubDate.isEmpty()) { entry_->setField(QStringLiteral("pub_year"), pubDate.left(4)); } QVariantList langArray = itemMap.value(QLatin1String("ContentInfo")).toMap() .value(QStringLiteral("Languages")).toMap() .value(QStringLiteral("DisplayValues")).toList(); QStringList langs; foreach(const QVariant& v, langArray) { langs += mapValue(v.toMap(), "DisplayValue"); } langs.removeDuplicates(); langs.removeAll(QString()); entry_->setField(QStringLiteral("language"), langs.join(FieldFormat::delimiterString())); QVariantMap imagesMap = info_.value(QLatin1String("Images")).toObject().toVariantMap(); switch(m_imageSize) { case SmallImage: entry_->setField(QStringLiteral("small-image"), mapValue(imagesMap, "Primary", "Small", "URL")); break; case MediumImage: entry_->setField(QStringLiteral("medium-image"), mapValue(imagesMap, "Primary", "Medium", "URL")); break; case LargeImage: entry_->setField(QStringLiteral("large-image"), mapValue(imagesMap, "Primary", "Large", "URL")); break; case NoImage: break; } if(optionalFields().contains(QStringLiteral("amazon"))) { entry_->setField(QStringLiteral("amazon"), mapValue(info_.toVariantMap(), "DetailPageURL")); } } void AmazonFetcher::parseTitle(Tellico::Data::EntryPtr entry_) { // assume that everything in brackets or parentheses is extra - QRegExp rx(QLatin1String("[\\(\\[](.*)[\\)\\]]")); - rx.setMinimal(true); + static const QRegularExpression rx(QLatin1String("[\\(\\[](.*?)[\\)\\]]")); QString title = entry_->field(QStringLiteral("title")); - int pos = rx.indexIn(title); - while(pos > -1) { - if(parseTitleToken(entry_, rx.cap(1))) { - title.remove(pos, rx.matchedLength()); + int pos = 0; + QRegularExpressionMatch match = rx.match(title, pos); + while(match.hasMatch()) { + pos = match.capturedStart(); + if(parseTitleToken(entry_, match.captured(1))) { + title.remove(match.capturedStart(), match.capturedLength()); --pos; // search again there } - pos = rx.indexIn(title, pos+1); + match = rx.match(title, pos+1); } - entry_->setField(QStringLiteral("title"), title.trimmed()); + entry_->setField(QStringLiteral("title"), title.simplified()); } bool AmazonFetcher::parseTitleToken(Tellico::Data::EntryPtr entry_, const QString& token_) { // myDebug() << "title token:" << token_; // if res = true, then the token gets removed from the title bool res = false; if(token_.indexOf(QLatin1String("widescreen"), 0, Qt::CaseInsensitive) > -1 || token_.indexOf(i18n("Widescreen"), 0, Qt::CaseInsensitive) > -1) { entry_->setField(QStringLiteral("widescreen"), QStringLiteral("true")); // res = true; leave it in the title } else if(token_.indexOf(QLatin1String("full screen"), 0, Qt::CaseInsensitive) > -1) { // skip, but go ahead and remove from title res = true; } else if(token_.indexOf(QLatin1String("import"), 0, Qt::CaseInsensitive) > -1) { // skip, but go ahead and remove from title res = true; } if(token_.indexOf(QLatin1String("blu-ray"), 0, Qt::CaseInsensitive) > -1) { entry_->setField(QStringLiteral("medium"), i18n("Blu-ray")); res = true; } else if(token_.indexOf(QLatin1String("hd dvd"), 0, Qt::CaseInsensitive) > -1) { entry_->setField(QStringLiteral("medium"), i18n("HD DVD")); res = true; } else if(token_.indexOf(QLatin1String("vhs"), 0, Qt::CaseInsensitive) > -1) { entry_->setField(QStringLiteral("medium"), i18n("VHS")); res = true; } if(token_.indexOf(QLatin1String("director's cut"), 0, Qt::CaseInsensitive) > -1 || token_.indexOf(i18n("Director's Cut"), 0, Qt::CaseInsensitive) > -1) { entry_->setField(QStringLiteral("directors-cut"), QStringLiteral("true")); // res = true; leave it in the title } if(token_.toLower() == QLatin1String("ntsc")) { entry_->setField(QStringLiteral("format"), i18n("NTSC")); res = true; } if(token_.toLower() == QLatin1String("dvd")) { entry_->setField(QStringLiteral("medium"), i18n("DVD")); res = true; } - static QRegExp regionRx(QLatin1String("Region [1-9]")); - if(regionRx.indexIn(token_) > -1) { - entry_->setField(QStringLiteral("region"), i18n(regionRx.cap(0).toUtf8().constData())); + static const QRegularExpression regionRx(QLatin1String("Region [1-9]")); + QRegularExpressionMatch match = regionRx.match(token_); + if(match.hasMatch()) { + entry_->setField(QStringLiteral("region"), i18n(match.captured().toUtf8().constData())); res = true; } if(entry_->collection()->type() == Data::Collection::Game) { Data::FieldPtr f = entry_->collection()->fieldByName(QStringLiteral("platform")); if(f && f->allowed().contains(token_)) { res = true; } } return res; } //static QString AmazonFetcher::defaultName() { return i18n("Amazon.com Web Services"); } QString AmazonFetcher::defaultIcon() { return favIcon("http://www.amazon.com"); } Tellico::StringHash AmazonFetcher::allOptionalFields() { StringHash hash; hash[QStringLiteral("keyword")] = i18n("Keywords"); hash[QStringLiteral("amazon")] = i18n("Amazon Link"); return hash; } Tellico::Fetch::ConfigWidget* AmazonFetcher::configWidget(QWidget* parent_) const { return new AmazonFetcher::ConfigWidget(parent_, this); } AmazonFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const AmazonFetcher* fetcher_/*=0*/) : Fetch::ConfigWidget(parent_) { QGridLayout* l = new QGridLayout(optionsWidget()); l->setSpacing(4); l->setColumnStretch(1, 10); int row = -1; QLabel* al = new QLabel(i18n("Registration is required for accessing the %1 data source. " "If you agree to the terms and conditions, sign " "up for an account, and enter your information below.", AmazonFetcher::defaultName(), QLatin1String("https://affiliate-program.amazon.com/gp/flex/advertising/api/sign-in.html")), optionsWidget()); al->setOpenExternalLinks(true); al->setWordWrap(true); ++row; l->addWidget(al, row, 0, 1, 2); // richtext gets weird with size al->setMinimumWidth(al->sizeHint().width()); QLabel* label = new QLabel(i18n("Access key: "), optionsWidget()); l->addWidget(label, ++row, 0); m_accessEdit = new QLineEdit(optionsWidget()); connect(m_accessEdit, &QLineEdit::textChanged, this, &ConfigWidget::slotSetModified); l->addWidget(m_accessEdit, row, 1); QString w = i18n("Access to data from Amazon.com requires an AWS Access Key ID and a Secret Key."); label->setWhatsThis(w); m_accessEdit->setWhatsThis(w); label->setBuddy(m_accessEdit); label = new QLabel(i18n("Secret key: "), optionsWidget()); l->addWidget(label, ++row, 0); m_secretKeyEdit = new QLineEdit(optionsWidget()); // m_secretKeyEdit->setEchoMode(QLineEdit::PasswordEchoOnEdit); connect(m_secretKeyEdit, &QLineEdit::textChanged, this, &ConfigWidget::slotSetModified); l->addWidget(m_secretKeyEdit, row, 1); label->setWhatsThis(w); m_secretKeyEdit->setWhatsThis(w); label->setBuddy(m_secretKeyEdit); label = new QLabel(i18n("Country: "), optionsWidget()); l->addWidget(label, ++row, 0); m_siteCombo = new GUI::ComboBox(optionsWidget()); for(int i = 0; i < XX; ++i) { const AmazonFetcher::SiteData& siteData = AmazonFetcher::siteData(i); QIcon icon(QStandardPaths::locate(QStandardPaths::GenericDataLocation, QStringLiteral("kf5/locale/countries/%1/flag.png").arg(siteData.country))); m_siteCombo->addItem(icon, siteData.countryName, i); m_siteCombo->model()->sort(0); } void (GUI::ComboBox::* activatedInt)(int) = &GUI::ComboBox::activated; connect(m_siteCombo, activatedInt, this, &ConfigWidget::slotSetModified); connect(m_siteCombo, activatedInt, this, &ConfigWidget::slotSiteChanged); l->addWidget(m_siteCombo, row, 1); w = i18n("Amazon.com provides data from several different localized sites. Choose the one " "you wish to use for this data source."); label->setWhatsThis(w); m_siteCombo->setWhatsThis(w); label->setBuddy(m_siteCombo); label = new QLabel(i18n("&Image size: "), optionsWidget()); l->addWidget(label, ++row, 0); m_imageCombo = new GUI::ComboBox(optionsWidget()); m_imageCombo->addItem(i18n("Small Image"), SmallImage); m_imageCombo->addItem(i18n("Medium Image"), MediumImage); m_imageCombo->addItem(i18n("Large Image"), LargeImage); m_imageCombo->addItem(i18n("No Image"), NoImage); connect(m_imageCombo, activatedInt, this, &ConfigWidget::slotSetModified); l->addWidget(m_imageCombo, row, 1); w = i18n("The cover image may be downloaded as well. However, too many large images in the " "collection may degrade performance."); label->setWhatsThis(w); m_imageCombo->setWhatsThis(w); label->setBuddy(m_imageCombo); label = new QLabel(i18n("&Associate's ID: "), optionsWidget()); l->addWidget(label, ++row, 0); m_assocEdit = new QLineEdit(optionsWidget()); void (QLineEdit::* textChanged)(const QString&) = &QLineEdit::textChanged; connect(m_assocEdit, textChanged, this, &ConfigWidget::slotSetModified); l->addWidget(m_assocEdit, row, 1); w = i18n("The associate's id identifies the person accessing the Amazon.com Web Services, and is included " "in any links to the Amazon.com site."); label->setWhatsThis(w); m_assocEdit->setWhatsThis(w); label->setBuddy(m_assocEdit); l->setRowStretch(++row, 10); if(fetcher_) { m_siteCombo->setCurrentData(fetcher_->m_site); m_accessEdit->setText(fetcher_->m_accessKey); m_secretKeyEdit->setText(fetcher_->m_secretKey); m_assocEdit->setText(fetcher_->m_assoc); m_imageCombo->setCurrentData(fetcher_->m_imageSize); } else { // defaults m_assocEdit->setText(QLatin1String(AMAZON_ASSOC_TOKEN)); m_imageCombo->setCurrentData(MediumImage); } addFieldsWidget(AmazonFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList()); KAcceleratorManager::manage(optionsWidget()); } void AmazonFetcher::ConfigWidget::saveConfigHook(KConfigGroup& config_) { int n = m_siteCombo->currentData().toInt(); config_.writeEntry("Site", n); QString s = m_accessEdit->text().trimmed(); if(!s.isEmpty()) { config_.writeEntry("AccessKey", s); } s = m_secretKeyEdit->text().trimmed(); if(!s.isEmpty()) { config_.writeEntry("SecretKey", s); } s = m_assocEdit->text().trimmed(); if(!s.isEmpty()) { config_.writeEntry("AssocToken", s); } n = m_imageCombo->currentData().toInt(); config_.writeEntry("Image Size", n); } QString AmazonFetcher::ConfigWidget::preferredName() const { return AmazonFetcher::siteData(m_siteCombo->currentData().toInt()).title; } void AmazonFetcher::ConfigWidget::slotSiteChanged() { emit signalName(preferredName()); } diff --git a/src/tests/amazonfetchertest.cpp b/src/tests/amazonfetchertest.cpp index 7aec5818..15af4a2e 100644 --- a/src/tests/amazonfetchertest.cpp +++ b/src/tests/amazonfetchertest.cpp @@ -1,539 +1,557 @@ /*************************************************************************** Copyright (C) 2010-2011 Robby Stephenson ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * * * ***************************************************************************/ #undef QT_NO_CAST_FROM_ASCII #include "amazonfetchertest.h" #include "../fetch/amazonfetcher.h" #include "../fetch/amazonrequest.h" #include "../fetch/messagelogger.h" #include "../collections/bookcollection.h" #include "../collections/musiccollection.h" #include "../collections/videocollection.h" #include "../collections/gamecollection.h" #include "../collectionfactory.h" #include "../entry.h" #include "../images/imagefactory.h" #include #include QTEST_GUILESS_MAIN( AmazonFetcherTest ) AmazonFetcherTest::AmazonFetcherTest() : AbstractFetcherTest(), m_hasConfigFile(false) , m_config(QFINDTESTDATA("tellicotest_private.config"), KConfig::SimpleConfig) { } void AmazonFetcherTest::initTestCase() { Tellico::RegisterCollection registerBook(Tellico::Data::Collection::Book, "book"); Tellico::RegisterCollection registerMusic(Tellico::Data::Collection::Album, "music"); Tellico::RegisterCollection registerVideo(Tellico::Data::Collection::Video, "mvideo"); Tellico::RegisterCollection registerGame(Tellico::Data::Collection::Game, "game"); Tellico::ImageFactory::init(); m_hasConfigFile = QFile::exists(QFINDTESTDATA("tellicotest_private.config")); QHash practicalRdf; practicalRdf.insert(QStringLiteral("title"), QStringLiteral("Practical RDF")); practicalRdf.insert(QStringLiteral("isbn"), QStringLiteral("0-596-00263-7")); practicalRdf.insert(QStringLiteral("author"), QStringLiteral("Shelley Powers")); practicalRdf.insert(QStringLiteral("binding"), QStringLiteral("Paperback")); practicalRdf.insert(QStringLiteral("publisher"), QStringLiteral("O'Reilly Media")); practicalRdf.insert(QStringLiteral("pages"), QStringLiteral("331")); QHash gloryRevealed; gloryRevealed.insert(QStringLiteral("title"), QStringLiteral("Glory Revealed II")); gloryRevealed.insert(QStringLiteral("medium"), QStringLiteral("Compact Disc")); // gloryRevealed.insert(QStringLiteral("artist"), QStringLiteral("Various Artists")); gloryRevealed.insert(QStringLiteral("label"), QStringLiteral("Reunion")); gloryRevealed.insert(QStringLiteral("year"), QStringLiteral("2009")); QHash incredibles; incredibles.insert(QStringLiteral("title"), QStringLiteral("Incredibles")); incredibles.insert(QStringLiteral("medium"), QStringLiteral("DVD")); // incredibles.insert(QStringLiteral("certification"), QStringLiteral("PG (USA)")); // incredibles.insert(QStringLiteral("studio"), QStringLiteral("Walt Disney Home Entertainment")); // incredibles.insert(QStringLiteral("year"), QStringLiteral("2004")); incredibles.insert(QStringLiteral("widescreen"), QStringLiteral("true")); incredibles.insert(QStringLiteral("director"), QStringLiteral("Brad Bird; Bud Luckey; Roger Gould")); QHash pacteDesLoups; pacteDesLoups.insert(QStringLiteral("title"), QStringLiteral("Le Pacte des Loups")); pacteDesLoups.insert(QStringLiteral("medium"), QStringLiteral("Blu-ray")); // pacteDesLoups.insert(QStringLiteral("region"), QStringLiteral("Region 2")); pacteDesLoups.insert(QStringLiteral("studio"), QStringLiteral("StudioCanal")); pacteDesLoups.insert(QStringLiteral("year"), QStringLiteral("2001")); pacteDesLoups.insert(QStringLiteral("director"), QStringLiteral("Christophe Gans")); // pacteDesLoups.insert(QStringLiteral("format"), QStringLiteral("PAL")); QHash petitPrinceCN; petitPrinceCN.insert(QStringLiteral("title"), QStringLiteral("小王子(65周年纪念版)")); petitPrinceCN.insert(QStringLiteral("author"), QStringLiteral("圣埃克絮佩里 (Saint-Exupery)")); m_fieldValues.insert(QStringLiteral("practicalRdf"), practicalRdf); m_fieldValues.insert(QStringLiteral("gloryRevealed"), gloryRevealed); m_fieldValues.insert(QStringLiteral("incredibles"), incredibles); m_fieldValues.insert(QStringLiteral("pacteDesLoups"), pacteDesLoups); m_fieldValues.insert(QStringLiteral("petitPrinceCN"), petitPrinceCN); } void AmazonFetcherTest::testTitle() { QFETCH(QString, locale); QFETCH(int, collType); QFETCH(QString, searchValue); QFETCH(QString, resultName); QString groupName = QStringLiteral("Amazon ") + locale; if(!m_hasConfigFile || !m_config.hasGroup(groupName)) { QSKIP("This test requires a config file with Amazon settings.", SkipAll); } KConfigGroup cg(&m_config, groupName); Tellico::Fetch::FetchRequest request(collType, Tellico::Fetch::Title, searchValue); Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::AmazonFetcher(this)); fetcher->readConfig(cg, cg.name()); Tellico::Data::EntryList results = DO_FETCH(fetcher, request); QVERIFY(!results.isEmpty()); Tellico::Data::EntryPtr entry = results.at(0); QHashIterator i(m_fieldValues.value(resultName)); while(i.hasNext()) { i.next(); // a known bug is CA video titles result in music results, so only title matches if(i.key() != QStringLiteral("title")) { QEXPECT_FAIL("CA video title", "CA video titles show music results for some reason", Continue); } QString result = entry->field(i.key()).toLower(); // several titles have edition info in the title if(collType == Tellico::Data::Collection::Video && i.key() == QStringLiteral("title") && (locale == QStringLiteral("CA") || locale == QStringLiteral("FR") || locale == QStringLiteral("ES") || locale == QStringLiteral("CN") || locale == QStringLiteral("IT") || locale == QStringLiteral("DE"))) { QVERIFY2(result.contains(i.value(), Qt::CaseInsensitive), qPrintable(i.key())); } else if(collType == Tellico::Data::Collection::Video && i.key() == QStringLiteral("year") && locale == QStringLiteral("FR")) { // france has no year for movie QCOMPARE(result, QString()); } else if(collType == Tellico::Data::Collection::Video && i.key() == QStringLiteral("medium") && (locale == QStringLiteral("ES") || locale == QStringLiteral("IT"))) { // ES and IT think it's a DVD QCOMPARE(result, QStringLiteral("dvd")); } else if(i.key() == QStringLiteral("pages") && (locale == QStringLiteral("UK") || locale == QStringLiteral("CA"))) { // UK and CA have different page count QCOMPARE(result, QStringLiteral("352")); } else if((i.key() == QStringLiteral("director") || i.key() == QStringLiteral("studio") || i.key() == QStringLiteral("year")) && (locale == QStringLiteral("ES") || locale == QStringLiteral("IT"))) { // ES and IT have no director or studio or year info QCOMPARE(result, QString()); } else { QCOMPARE(result, i.value().toLower()); } } QVERIFY(!entry->field(QStringLiteral("cover")).isEmpty()); QVERIFY(!entry->field(QStringLiteral("cover")).contains(QLatin1Char('/'))); } void AmazonFetcherTest::testTitle_data() { QTest::addColumn("locale"); QTest::addColumn("collType"); QTest::addColumn("searchValue"); QTest::addColumn("resultName"); QTest::newRow("US book title") << QStringLiteral("US") << static_cast(Tellico::Data::Collection::Book) << QStringLiteral("Practical RDF") << QStringLiteral("practicalRdf"); QTest::newRow("UK book title") << QStringLiteral("UK") << static_cast(Tellico::Data::Collection::Book) << QStringLiteral("Practical RDF") << QStringLiteral("practicalRdf"); // QTest::newRow("DE") << QString::fromLatin1("DE"); // QTest::newRow("JP") << QString::fromLatin1("JP"); // QTest::newRow("FR") << QString::fromLatin1("FR"); QTest::newRow("CA book title") << QStringLiteral("CA") << static_cast(Tellico::Data::Collection::Book) << QStringLiteral("Practical RDF") << QStringLiteral("practicalRdf"); QTest::newRow("CN book title") << QStringLiteral("CN") << static_cast(Tellico::Data::Collection::Book) << QStringLiteral("小王子(65周年纪念版)") << QStringLiteral("petitPrinceCN"); // a known bug is CA video titles result in music results, so only title matches // QTest::newRow("CA video title") << QString::fromLatin1("CA") // << static_cast(Tellico::Data::Collection::Video) // << QString::fromLatin1("Le Pacte des Loups") // << QString::fromLatin1("pacteDesLoups"); QTest::newRow("FR video title") << QStringLiteral("FR") << static_cast(Tellico::Data::Collection::Video) << QStringLiteral("Le Pacte des Loups") << QStringLiteral("pacteDesLoups"); QTest::newRow("ES video title") << QStringLiteral("ES") << static_cast(Tellico::Data::Collection::Video) << QStringLiteral("Le Pacte des Loups") << QStringLiteral("pacteDesLoups"); QTest::newRow("IT video title") << QStringLiteral("IT") << static_cast(Tellico::Data::Collection::Video) << QStringLiteral("Le Pacte des Loups") << QStringLiteral("pacteDesLoups"); } void AmazonFetcherTest::testTitleVideoGame() { QString groupName = QStringLiteral("Amazon US"); if(!m_hasConfigFile || !m_config.hasGroup(groupName)) { QSKIP("This test requires a config file with Amazon settings.", SkipAll); } KConfigGroup cg(&m_config, groupName); Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Game, Tellico::Fetch::Title, QStringLiteral("Ghostbusters Story Pack - LEGO Dimensions")); Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::AmazonFetcher(this)); fetcher->readConfig(cg, cg.name()); Tellico::Data::EntryList results = DO_FETCH(fetcher, request); QVERIFY(!results.isEmpty()); Tellico::Data::EntryPtr entry = results.at(0); QVERIFY(entry); QCOMPARE(entry->field("title"), QStringLiteral("Ghostbusters Story Pack - LEGO Dimensions")); QCOMPARE(entry->field("publisher"), QStringLiteral("Warner Home Video - Games")); // the E10+ ESRB rating was added to Tellico in 2017 in version 3.0.1 QCOMPARE(entry->field("certification"), QStringLiteral("Everyone 10+")); } void AmazonFetcherTest::testIsbn() { QFETCH(QString, locale); QFETCH(QString, searchValue); QFETCH(QString, resultName); QString groupName = QStringLiteral("Amazon ") + locale; if(!m_hasConfigFile || !m_config.hasGroup(groupName)) { QSKIP("This test requires a config file with Amazon settings.", SkipAll); } KConfigGroup cg(&m_config, groupName); // also testing multiple values Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Book, Tellico::Fetch::ISBN, searchValue); Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::AmazonFetcher(this)); fetcher->readConfig(cg, cg.name()); Tellico::Data::EntryList results = DO_FETCH(fetcher, request); QCOMPARE(results.size(), 2); Tellico::Data::EntryPtr entry = results.at(0); QHashIterator i(m_fieldValues.value(resultName)); while(i.hasNext()) { i.next(); QString result = entry->field(i.key()).toLower(); if(i.key() == QStringLiteral("pages") && (locale == QStringLiteral("UK") || locale == QStringLiteral("CA"))) { // UK and CA have different page count QCOMPARE(result, QStringLiteral("352")); } else { QCOMPARE(result, i.value().toLower()); } } QVERIFY(!entry->field(QStringLiteral("cover")).isEmpty()); QVERIFY(!entry->field(QStringLiteral("cover")).contains(QLatin1Char('/'))); } void AmazonFetcherTest::testIsbn_data() { QTest::addColumn("locale"); QTest::addColumn("searchValue"); QTest::addColumn("resultName"); QTest::newRow("US isbn") << QStringLiteral("US") << QStringLiteral("0-596-00263-7; 978-1-59059-831-3") << QStringLiteral("practicalRdf"); QTest::newRow("UK isbn") << QStringLiteral("UK") << QStringLiteral("0-596-00263-7; 978-1-59059-831-3") << QStringLiteral("practicalRdf"); // QTest::newRow("DE") << QString::fromLatin1("DE"); // QTest::newRow("JP") << QString::fromLatin1("JP"); // QTest::newRow("FR") << QString::fromLatin1("FR"); QTest::newRow("CA isbn") << QStringLiteral("CA") << QStringLiteral("0-596-00263-7; 978-1-59059-831-3") << QStringLiteral("practicalRdf"); /* QTest::newRow("CN isbn") << QString::fromLatin1("CN") << QString::fromLatin1("7511305202") << QString::fromLatin1("petitPrinceCN"); */ } void AmazonFetcherTest::testUpc() { QFETCH(QString, locale); QFETCH(int, collType); QFETCH(QString, searchValue); QFETCH(QString, resultName); QString groupName = QStringLiteral("Amazon ") + locale; if(!m_hasConfigFile || !m_config.hasGroup(groupName)) { QSKIP("This test requires a config file with Amazon settings.", SkipAll); } KConfigGroup cg(&m_config, groupName); Tellico::Fetch::FetchRequest request(collType, Tellico::Fetch::UPC, searchValue); Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::AmazonFetcher(this)); fetcher->readConfig(cg, cg.name()); Tellico::Data::EntryList results = DO_FETCH(fetcher, request); QVERIFY(!results.isEmpty()); Tellico::Data::EntryPtr entry = results.at(0); QHashIterator i(m_fieldValues.value(resultName)); while(i.hasNext()) { i.next(); QString result = entry->field(i.key()).toLower(); // exception for UK label different than US // and FR title having edition info if((i.key() == QStringLiteral("label") && locale == QStringLiteral("UK")) || (i.key() == QStringLiteral("title"))) { QVERIFY(result.contains(i.value(), Qt::CaseInsensitive)); } else if(i.key() == QStringLiteral("year") && locale == QStringLiteral("FR")) { // france has no year for movie QCOMPARE(result, QString()); } else { QCOMPARE(result, i.value().toLower()); } } QVERIFY(!entry->field(QStringLiteral("cover")).isEmpty()); QVERIFY(!entry->field(QStringLiteral("cover")).contains(QLatin1Char('/'))); if(collType == Tellico::Data::Collection::Album) { QVERIFY(!entry->field(QStringLiteral("genre")).isEmpty()); QVERIFY(!entry->field(QStringLiteral("track")).isEmpty()); } else if(collType == Tellico::Data::Collection::Video) { QVERIFY(!entry->field(QStringLiteral("cast")).isEmpty()); } } void AmazonFetcherTest::testUpc_data() { QTest::addColumn("locale"); QTest::addColumn("collType"); QTest::addColumn("searchValue"); QTest::addColumn("resultName"); QTest::newRow("US music upc") << QStringLiteral("US") << static_cast(Tellico::Data::Collection::Album) << QStringLiteral("602341013727") << QStringLiteral("gloryRevealed"); // non-US should work with or without the initial 0 country code QTest::newRow("UK music upc1") << QStringLiteral("UK") << static_cast(Tellico::Data::Collection::Album) << QStringLiteral("602341013727") << QStringLiteral("gloryRevealed"); QTest::newRow("UK music upc2") << QStringLiteral("UK") << static_cast(Tellico::Data::Collection::Album) << QStringLiteral("0602341013727") << QStringLiteral("gloryRevealed"); QTest::newRow("CA music upc") << QStringLiteral("CA") << static_cast(Tellico::Data::Collection::Album) << QStringLiteral("0602341013727") << QStringLiteral("gloryRevealed"); QTest::newRow("US movie upc") << QStringLiteral("US") << static_cast(Tellico::Data::Collection::Video) << QStringLiteral("786936244250") << QStringLiteral("incredibles"); QTest::newRow("UK movie upc") << QStringLiteral("UK") << static_cast(Tellico::Data::Collection::Video) << QStringLiteral("0786936244250") << QStringLiteral("incredibles"); QTest::newRow("CA movie upc") << QStringLiteral("CA") << static_cast(Tellico::Data::Collection::Video) << QStringLiteral("0786936244250") << QStringLiteral("incredibles"); QTest::newRow("FR movie upc") << QStringLiteral("FR") << static_cast(Tellico::Data::Collection::Video) << QStringLiteral("5050582560985") << QStringLiteral("pacteDesLoups"); } void AmazonFetcherTest::testRequest() { // from aws-sig-v4-test-suite/post-vanilla Tellico::Fetch::AmazonRequest req("AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"); req.setHost("example.amazonaws.com"); req.m_headers.insert("host", req.m_host); req.m_amzDate = "20150830T123600Z"; req.m_headers.insert("x-amz-date", req.m_amzDate); req.m_path = "/"; QByteArray res1("POST\n/\n\nhost:example.amazonaws.com\nx-amz-date:20150830T123600Z\n\nhost;x-amz-date\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); QCOMPARE(req.prepareCanonicalRequest(""), res1); req.m_region = "us-east-1"; req.m_service = "service"; QByteArray res2("AWS4-HMAC-SHA256\n20150830T123600Z\n20150830/us-east-1/service/aws4_request\n553f88c9e4d10fc9e109e2aeb65f030801b70c2f6468faca261d401ae622fc87"); QCOMPARE(req.prepareStringToSign(res1), res2); QByteArray res3("AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b"); QCOMPARE(req.buildAuthorizationString(req.calculateSignature(res2)), res3); QByteArray res4("com.amazon.paapi5.v1.servicev1.SearchItems"); QCOMPARE(req.targetOperation(), res4); } void AmazonFetcherTest::testPayload() { QString groupName = QStringLiteral("Amazon US"); if(!m_hasConfigFile || !m_config.hasGroup(groupName)) { QSKIP("This test requires a config file with Amazon settings.", SkipAll); } KConfigGroup cg(&m_config, groupName); Tellico::Fetch::AmazonFetcher* fetcher = new Tellico::Fetch::AmazonFetcher(this); fetcher->readConfig(cg, cg.name()); Tellico::Fetch::FetchRequest req(Tellico::Data::Collection::Book, Tellico::Fetch::UPC, "717356278525"); QByteArray payload = fetcher->requestPayload(req); QByteArray res1("{\ \"Keywords\":\"717356278525\",\ \"Operation\":\"SearchItems\",\ \"PartnerTag\":\"tellico-20\",\ \"PartnerType\":\"Associates\",\ \"Resources\":[\"ItemInfo.Title\",\"ItemInfo.ExternalIds\",\"Images.Primary.Medium\"],\ \"SearchIndex\":\"Books\"\ }"); QCOMPARE(payload, res1); } void AmazonFetcherTest::testError() { Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Book, Tellico::Fetch::UPC, "717356278525"); Tellico::Fetch::AmazonFetcher* f = new Tellico::Fetch::AmazonFetcher(this); Tellico::Fetch::Fetcher::Ptr fetcher(f); Tellico::Fetch::MessageLogger* logger = new Tellico::Fetch::MessageLogger; f->setMessageHandler(logger); f->m_site = Tellico::Fetch::AmazonFetcher::US; f->m_accessKey = QStringLiteral("test"); f->m_secretKey = QStringLiteral("test"); f->m_testResultsFile = QFINDTESTDATA("data/amazon-paapi-error1.json"); Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1); QVERIFY(!logger->errorList.isEmpty()); QCOMPARE(logger->errorList[0], QStringLiteral("The Access Key ID or security token included in the request is invalid.")); } void AmazonFetcherTest::testUpc1() { QString groupName = QStringLiteral("Amazon US"); if(!m_hasConfigFile || !m_config.hasGroup(groupName)) { QSKIP("This test requires a config file with Amazon settings.", SkipAll); } KConfigGroup cg(&m_config, groupName); Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Book, Tellico::Fetch::UPC, "717356278525"); Tellico::Fetch::AmazonFetcher* f = new Tellico::Fetch::AmazonFetcher(this); Tellico::Fetch::Fetcher::Ptr fetcher(f); fetcher->readConfig(cg, cg.name()); f->m_testResultsFile = QFINDTESTDATA("data/amazon-paapi-upc1.json"); Tellico::Data::EntryList results = DO_FETCH(fetcher, request); QCOMPARE(results.size(), 1); Tellico::Data::EntryPtr entry = results.at(0); QVERIFY(entry); QCOMPARE(entry->field("title"), QStringLiteral("Harry Potter Paperback Box Set (Books 1-7)")); QCOMPARE(entry->field("isbn"), QStringLiteral("0545162076")); QVERIFY(!entry->field(QStringLiteral("cover")).isEmpty()); QVERIFY(!entry->field(QStringLiteral("cover")).contains(QLatin1Char('/'))); } void AmazonFetcherTest::testUpc2() { QString groupName = QStringLiteral("Amazon US"); if(!m_hasConfigFile || !m_config.hasGroup(groupName)) { QSKIP("This test requires a config file with Amazon settings.", SkipAll); } KConfigGroup cg(&m_config, groupName); Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Book, Tellico::Fetch::UPC, "717356278525; 842776102270"); Tellico::Fetch::AmazonFetcher* f = new Tellico::Fetch::AmazonFetcher(this); Tellico::Fetch::Fetcher::Ptr fetcher(f); fetcher->readConfig(cg, cg.name()); QByteArray payload = f->requestPayload(request); // verify the format of the multiple UPC keyword QVERIFY(payload.contains("\"Keywords\":\"717356278525|842776102270\"")); f->m_testResultsFile = QFINDTESTDATA("data/amazon-paapi-upc2.json"); Tellico::Data::EntryList results = DO_FETCH(fetcher, request); QCOMPARE(results.size(), 2); Tellico::Data::EntryPtr entry = results.at(0); QVERIFY(entry); QCOMPARE(entry->field("title"), QStringLiteral("Harry Potter Paperback Box Set (Books 1-7)")); QCOMPARE(entry->field("isbn"), QStringLiteral("0545162076")); QVERIFY(!entry->field(QStringLiteral("cover")).isEmpty()); QVERIFY(!entry->field(QStringLiteral("cover")).contains(QLatin1Char('/'))); } // from https://github.com/dkam/paapi/blob/master/test/data/get_item_no_author.json void AmazonFetcherTest::testNoAuthor() { QString groupName = QStringLiteral("Amazon US"); if(!m_hasConfigFile || !m_config.hasGroup(groupName)) { QSKIP("This test requires a config file with Amazon settings.", SkipAll); } KConfigGroup cg(&m_config, groupName); Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Book, Tellico::Fetch::UPC, "717356278525; 842776102270"); Tellico::Fetch::AmazonFetcher* f = new Tellico::Fetch::AmazonFetcher(this); Tellico::Fetch::Fetcher::Ptr fetcher(f); cg.writeEntry("Image Size", 0); cg.markAsClean(); fetcher->readConfig(cg, cg.name()); f->m_testResultsFile = QFINDTESTDATA("data/amazon-paapi-no-author.json"); Tellico::Data::EntryList results = DO_FETCH(fetcher, request); QCOMPARE(results.size(), 1); Tellico::Data::EntryPtr entry = results.at(0); QVERIFY(entry); QCOMPARE(entry->field("title"), QStringLiteral("Muscle Car Mania: 100 legendary Australian motoring stories (Motoring Series)")); QCOMPARE(entry->field("isbn"), QStringLiteral("1921878657")); QCOMPARE(entry->field("pages"), QStringLiteral("224")); QCOMPARE(entry->field("language"), QStringLiteral("English")); QCOMPARE(entry->field("pub_year"), QStringLiteral("2013")); + QCOMPARE(entry->field("amazon"), QStringLiteral("https://www.amazon.com/dp/1921878657?tag=bookie09-20&linkCode=ogi&th=1&psc=1")); QVERIFY(entry->field(QStringLiteral("cover")).isEmpty()); // because image size as NoImage } + +void AmazonFetcherTest::testTitleParsing() { + Tellico::Data::CollPtr coll(new Tellico::Data::VideoCollection(true)); + Tellico::Data::EntryPtr entry(new Tellico::Data::Entry(coll)); + entry->setField(QStringLiteral("title"), QStringLiteral("title1 [DVD] (Widescreen) (NTSC) [Region 1]")); + + Tellico::Fetch::AmazonFetcher* f = new Tellico::Fetch::AmazonFetcher(this); + Tellico::Fetch::Fetcher::Ptr fetcher(f); + + f->parseTitle(entry); + // the fetcher leaves widescreen in the title but adds the field value + QCOMPARE(entry->field("title"), QStringLiteral("title1 (Widescreen)")); + QCOMPARE(entry->field("medium"), QStringLiteral("DVD")); + QCOMPARE(entry->field("widescreen"), QStringLiteral("true")); + QCOMPARE(entry->field("format"), QStringLiteral("NTSC")); + QCOMPARE(entry->field("region"), QStringLiteral("Region 1")); +} diff --git a/src/tests/amazonfetchertest.h b/src/tests/amazonfetchertest.h index 50247b64..0812022e 100644 --- a/src/tests/amazonfetchertest.h +++ b/src/tests/amazonfetchertest.h @@ -1,59 +1,60 @@ /*************************************************************************** Copyright (C) 2010-2011 Robby Stephenson ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * * * ***************************************************************************/ #ifndef AMAZONFETCHERTEST_H #define AMAZONFETCHERTEST_H #include "abstractfetchertest.h" #include class AmazonFetcherTest : public AbstractFetcherTest { Q_OBJECT public: AmazonFetcherTest(); private Q_SLOTS: void initTestCase(); void testTitle(); void testTitle_data(); void testTitleVideoGame(); void testIsbn(); void testIsbn_data(); void testUpc(); void testUpc_data(); void testRequest(); void testPayload(); void testError(); void testUpc1(); void testUpc2(); void testNoAuthor(); + void testTitleParsing(); private: bool m_hasConfigFile; KConfig m_config; QHash > m_fieldValues; }; #endif