diff --git a/src/collection.cpp b/src/collection.cpp index fb0ee59d..147a4497 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -1,880 +1,880 @@ /*************************************************************************** Copyright (C) 2001-2009 Robby Stephenson ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * * * ***************************************************************************/ #include "collection.h" #include "field.h" #include "entry.h" #include "entrygroup.h" #include "derivedvalue.h" #include "fieldformat.h" #include "utils/string_utils.h" #include "utils/stringset.h" #include "entrycomparison.h" #include "tellico_debug.h" #include #include using namespace Tellico; using Tellico::Data::Collection; const QString Collection::s_peopleGroupName = QStringLiteral("_people"); Collection::Collection(const QString& title_) : QObject(), QSharedData(), m_nextEntryId(1), m_title(title_), m_trackGroups(false) { m_id = getID(); } Collection::Collection(bool addDefaultFields_, const QString& title_) : QObject(), QSharedData(), m_nextEntryId(1), m_title(title_), m_trackGroups(false) { if(m_title.isEmpty()) { m_title = i18n("My Collection"); } m_id = getID(); if(addDefaultFields_) { addField(Field::createDefaultField(Field::IDField)); addField(Field::createDefaultField(Field::TitleField)); addField(Field::createDefaultField(Field::CreatedDateField)); addField(Field::createDefaultField(Field::ModifiedDateField)); } } Collection::~Collection() { // maybe we should just call clear() ? foreach(EntryGroupDict* dict, m_entryGroupDicts) { qDeleteAll(*dict); } qDeleteAll(m_entryGroupDicts); m_entryGroupDicts.clear(); } bool Collection::addFields(Tellico::Data::FieldList list_) { bool success = true; foreach(FieldPtr field, list_) { success &= addField(field); } return success; } bool Collection::addField(Tellico::Data::FieldPtr field_) { Q_ASSERT(field_); if(!field_) { return false; } // this essentially checks for duplicates if(hasField(field_->name())) { myDebug() << "replacing" << field_->name() << "in collection" << m_title; removeField(fieldByName(field_->name()), true); } m_fields.append(field_); m_fieldByName.insert(field_->name(), field_.data()); m_fieldByTitle.insert(field_->title(), field_.data()); if(field_->formatType() == FieldFormat::FormatName) { m_peopleFields.append(field_); // list of people attributes if(m_peopleFields.count() > 1) { // the second time that a person field is added, add a "pseudo-group" for people if(!m_entryGroupDicts.contains(s_peopleGroupName)) { EntryGroupDict* d = new EntryGroupDict(); m_entryGroupDicts.insert(s_peopleGroupName, d); m_entryGroups.prepend(s_peopleGroupName); } } } if(field_->type() == Field::Image) { m_imageFields.append(field_); } if(!field_->category().isEmpty() && !m_fieldCategories.contains(field_->category())) { m_fieldCategories << field_->category(); } if(field_->hasFlag(Field::AllowGrouped)) { // m_entryGroupsDicts autoDeletes each QDict when the Collection d'tor is called EntryGroupDict* dict = new EntryGroupDict(); m_entryGroupDicts.insert(field_->name(), dict); // cache the possible groups of entries m_entryGroups << field_->name(); } if(m_defaultGroupField.isEmpty() && field_->hasFlag(Field::AllowGrouped)) { m_defaultGroupField = field_->name(); } if(field_->hasFlag(Field::Derived)) { DerivedValue dv(field_); if(dv.isRecursive(this)) { field_->setProperty(QStringLiteral("template"), QString()); } } // refresh all dependent fields, in case one references this new one foreach(FieldPtr existingField, m_fields) { if(existingField->hasFlag(Field::Derived)) { emit signalRefreshField(existingField); } } return true; } bool Collection::mergeField(Tellico::Data::FieldPtr newField_) { if(!newField_) { return false; } FieldPtr currField = fieldByName(newField_->name()); if(!currField) { // does not exist in current collection, add it Data::FieldPtr f(new Field(*newField_)); bool success = addField(f); emit mergeAddedField(CollPtr(this), f); return success; } if(newField_->type() == Field::Table2) { newField_->setType(Data::Field::Table); newField_->setProperty(QStringLiteral("columns"), QStringLiteral("2")); } // the original field type is kept if(currField->type() != newField_->type()) { myDebug() << "skipping, field type mismatch for " << currField->title(); return false; } // if field is a Choice, then make sure all values are there if(currField->type() == Field::Choice && currField->allowed() != newField_->allowed()) { QStringList allowed = currField->allowed(); const QStringList& newAllowed = newField_->allowed(); for(QStringList::ConstIterator it = newAllowed.begin(); it != newAllowed.end(); ++it) { if(!allowed.contains(*it)) { allowed.append(*it); } } currField->setAllowed(allowed); } // don't change original format flags // don't change original category // add new description if current is empty if(currField->description().isEmpty()) { currField->setDescription(newField_->description()); } // if new field has additional extended properties, add those for(StringMap::const_iterator it = newField_->propertyList().begin(); it != newField_->propertyList().end(); ++it) { const QString propName = it.key(); const QString currValue = currField->property(propName); if(currValue.isEmpty()) { currField->setProperty(propName, it.value()); } else if (it.value() != currValue) { if(currField->type() == Field::URL && propName == QLatin1String("relative")) { myWarning() << "relative URL property does not match for " << currField->name(); } else if((currField->type() == Field::Table && propName == QLatin1String("columns")) || (currField->type() == Field::Rating && propName == QLatin1String("maximum"))) { bool ok; uint currNum = Tellico::toUInt(currValue, &ok); uint newNum = Tellico::toUInt(it.value(), &ok); if(newNum > currNum) { // bigger values currField->setProperty(propName, QString::number(newNum)); } } else if(currField->type() == Field::Rating && propName == QLatin1String("minimum")) { bool ok; uint currNum = Tellico::toUInt(currValue, &ok); uint newNum = Tellico::toUInt(it.value(), &ok); if(newNum < currNum) { // smaller values currField->setProperty(propName, QString::number(newNum)); } } } if(propName == QLatin1String("template") && currField->hasFlag(Field::Derived)) { DerivedValue dv(currField); if(dv.isRecursive(this)) { currField->setProperty(QStringLiteral("template"), QString()); } } } // combine flags currField->setFlags(currField->flags() | newField_->flags()); return true; } // be really careful with these field pointers, try not to call too many other functions // which may depend on the field list bool Collection::modifyField(Tellico::Data::FieldPtr newField_) { if(!newField_) { return false; } // myDebug() << "; // the field name never changes const QString fieldName = newField_->name(); FieldPtr oldField = fieldByName(fieldName); if(!oldField) { myDebug() << "no field named " << fieldName; return false; } // update name dict m_fieldByName.insert(fieldName, newField_.data()); // update titles const QString oldTitle = oldField->title(); const QString newTitle = newField_->title(); if(oldTitle == newTitle) { m_fieldByTitle.insert(newTitle, newField_.data()); } else { m_fieldByTitle.remove(oldTitle); m_fieldByTitle.insert(newTitle, newField_.data()); } // now replace the field pointer in the list int pos = m_fields.indexOf(oldField); if(pos > -1) { m_fields.replace(pos, newField_); } else { myDebug() << "no index found!"; return false; } // update category list. if(oldField->category() != newField_->category()) { m_fieldCategories.clear(); foreach(FieldPtr it, m_fields) { // add category if it's not in the list yet if(!it->category().isEmpty() && !m_fieldCategories.contains(it->category())) { m_fieldCategories += it->category(); } } } if(newField_->hasFlag(Field::Derived)) { DerivedValue dv(newField_); if(dv.isRecursive(this)) { newField_->setProperty(QStringLiteral("template"), QString()); } } // keep track of if the entry groups will need to be reset bool resetGroups = false; // if format is different, go ahead and invalidate all formatted entry values if(oldField->formatType() != newField_->formatType()) { // invalidate cached format strings of all entry attributes of this name foreach(EntryPtr entry, m_entries) { entry->invalidateFormattedFieldValue(fieldName); } resetGroups = true; } // check to see if the people "pseudo-group" needs to be updated // only if only one of the two is a name bool wasPeople = oldField->formatType() == FieldFormat::FormatName; bool isPeople = newField_->formatType() == FieldFormat::FormatName; if(wasPeople) { m_peopleFields.removeAll(oldField); if(!isPeople) { resetGroups = true; } } if(isPeople) { // if there's more than one people field and no people dict exists yet, add it if(m_peopleFields.count() > 1 && !m_entryGroupDicts.contains(s_peopleGroupName)) { EntryGroupDict* d = new EntryGroupDict(); m_entryGroupDicts.insert(s_peopleGroupName, d); // put it at the top of the list m_entryGroups.prepend(s_peopleGroupName); } m_peopleFields.append(newField_); if(!wasPeople) { resetGroups = true; } } bool wasGrouped = oldField->hasFlag(Field::AllowGrouped); bool isGrouped = newField_->hasFlag(Field::AllowGrouped); if(wasGrouped) { if(!isGrouped) { // in order to keep list in the same order, don't remove unless new field is not groupable m_entryGroups.removeAll(fieldName); delete m_entryGroupDicts.take(fieldName); // no auto-delete here myDebug() << "no longer grouped: " << fieldName; resetGroups = true; } else { // don't do this, it wipes out the old groups! // m_entryGroupDicts.replace(fieldName, new EntryGroupDict()); } } else if(isGrouped) { EntryGroupDict* d = new EntryGroupDict(); m_entryGroupDicts.insert(fieldName, d); if(!wasGrouped) { // cache the possible groups of entries m_entryGroups << fieldName; } resetGroups = true; } if(oldField->type() == Field::Image) { m_imageFields.removeAll(oldField); } if(newField_->type() == Field::Image) { m_imageFields.append(newField_); } if(resetGroups) { // myLog() << "invalidating groups"; invalidateGroups(); } // now to update all entries if the field is a derived value and the template changed if(newField_->hasFlag(Field::Derived) && oldField->property(QStringLiteral("template")) != newField_->property(QStringLiteral("template"))) { emit signalRefreshField(newField_); } return true; } bool Collection::removeField(const QString& name_, bool force_) { return removeField(fieldByName(name_), force_); } // force allows me to force the deleting of the title field if I need to bool Collection::removeField(Tellico::Data::FieldPtr field_, bool force_/*=false*/) { if(!field_ || !m_fields.contains(field_)) { if(field_) { myDebug() << "can't delete field:" << field_->name(); } return false; } // myDebug() << "name = " << field_->name(); // can't delete the title field if((field_->hasFlag(Field::NoDelete)) && !force_) { return false; } foreach(EntryPtr entry, m_entries) { // setting the fields to an empty string removes the value from the entry's list entry->setField(field_, QString()); } bool success = true; if(field_->formatType() == FieldFormat::FormatName) { m_peopleFields.removeAll(field_); } if(field_->type() == Field::Image) { m_imageFields.removeAll(field_); } m_fieldByName.remove(field_->name()); m_fieldByTitle.remove(field_->title()); if(fieldsByCategory(field_->category()).count() == 1) { m_fieldCategories.removeAll(field_->category()); } if(field_->hasFlag(Field::AllowGrouped)) { EntryGroupDict* dict = m_entryGroupDicts.take(field_->name()); qDeleteAll(*dict); m_entryGroups.removeAll(field_->name()); if(field_->name() == m_defaultGroupField && !m_entryGroups.isEmpty()) { setDefaultGroupField(m_entryGroups.first()); } } m_fields.removeAll(field_); // refresh all dependent fields, rather lazy, but there's // likely to be weird effects when checking dependent fields // while removing one, so refresh all of them foreach(FieldPtr field, m_fields) { if(field->hasFlag(Field::Derived)) { emit signalRefreshField(field); } } return success; } void Collection::reorderFields(const Tellico::Data::FieldList& list_) { // assume the lists have the same pointers! m_fields = list_; // also reset category list, since the order may have changed m_fieldCategories.clear(); foreach(FieldPtr field, m_fields) { if(!field->category().isEmpty() && !m_fieldCategories.contains(field->category())) { m_fieldCategories << field->category(); } } } void Collection::addEntries(const Tellico::Data::EntryList& entries_) { if(entries_.isEmpty()) { return; } foreach(EntryPtr entry, entries_) { if(!entry) { Q_ASSERT(entry); continue; } bool foster = false; if(this != entry->collection().data()) { entry->setCollection(CollPtr(this)); foster = true; } m_entries.append(entry); // myDebug() << "added entry (" << entry->title() << ")" << entry->id(); if(entry->id() >= m_nextEntryId) { m_nextEntryId = entry->id() + 1; } else if(entry->id() == -1) { entry->setId(m_nextEntryId); ++m_nextEntryId; } else if(m_entryById.contains(entry->id())) { if(!foster) { myDebug() << "the collection already has an entry with id = " << entry->id(); } entry->setId(m_nextEntryId); ++m_nextEntryId; } m_entryById.insert(entry->id(), entry.data()); if(hasField(QStringLiteral("cdate")) && entry->field(QStringLiteral("cdate")).isEmpty()) { entry->setField(QStringLiteral("cdate"), QDate::currentDate().toString(Qt::ISODate)); } if(hasField(QStringLiteral("mdate")) && entry->field(QStringLiteral("mdate")).isEmpty()) { entry->setField(QStringLiteral("mdate"), QDate::currentDate().toString(Qt::ISODate)); } } if(m_trackGroups) { populateCurrentDicts(entries_, fieldNames()); } } void Collection::removeEntriesFromDicts(const Tellico::Data::EntryList& entries_, const QStringList& fields_) { QSet modifiedGroups; foreach(EntryPtr entry, entries_) { // need a copy of the vector since it gets changed QList groups = entry->groups(); foreach(EntryGroup* group, groups) { // only clear groups for the modified fields, skip the others // also clear for all derived values, just in case if(!fields_.contains(group->fieldName()) && hasField(group->fieldName()) && !fieldByName(group->fieldName())->hasFlag(Field::Derived)) { continue; } if(entry->removeFromGroup(group)) { modifiedGroups.insert(group); } if(group->isEmpty() && !m_groupsToDelete.contains(group)) { m_groupsToDelete.push_back(group); } } } if(!modifiedGroups.isEmpty()) { - emit signalGroupsModified(CollPtr(this), modifiedGroups.toList()); + emit signalGroupsModified(CollPtr(this), modifiedGroups.values()); } } // this function gets called whenever an entry is modified. Its purpose is to keep the // groupDicts current. It first removes the entry from every group to which it belongs, // then it repopulates the dicts with the entry's fields void Collection::updateDicts(const Tellico::Data::EntryList& entries_, const QStringList& fields_) { if(entries_.isEmpty() || !m_trackGroups) { return; } QStringList modifiedFields = fields_; if(modifiedFields.isEmpty()) { // myDebug() << "updating all fields"; modifiedFields = fieldNames(); } removeEntriesFromDicts(entries_, modifiedFields); populateCurrentDicts(entries_, modifiedFields); cleanGroups(); } bool Collection::removeEntries(const Tellico::Data::EntryList& vec_) { if(vec_.isEmpty()) { return false; } removeEntriesFromDicts(vec_, fieldNames()); bool success = true; foreach(EntryPtr entry, vec_) { m_entryById.remove(entry->id()); m_entries.removeAll(entry); } cleanGroups(); return success; } Tellico::Data::FieldList Collection::fieldsByCategory(const QString& cat_) { #ifndef NDEBUG if(!m_fieldCategories.contains(cat_)) { myDebug() << cat_ << "' is not in category list"; } #endif if(cat_.isEmpty()) { myDebug() << "empty category!"; return FieldList(); } FieldList list; foreach(FieldPtr field, m_fields) { if(field->category() == cat_) { list.append(field); } } return list; } QString Collection::fieldNameByTitle(const QString& title_) const { if(title_.isEmpty()) { return QString(); } FieldPtr f = fieldByTitle(title_); if(!f) { // might happen in MainWindow::saveCollectionOptions return QString(); } return f->name(); } QStringList Collection::fieldNames() const { return m_fieldByName.keys(); } QStringList Collection::fieldTitles() const { return m_fieldByTitle.keys(); } QString Collection::fieldTitleByName(const QString& name_) const { if(name_.isEmpty()) { return QString(); } FieldPtr f = fieldByName(name_); if(!f) { myWarning() << "no field named " << name_; return QString(); } return f->title(); } QStringList Collection::valuesByFieldName(const QString& name_) const { if(name_.isEmpty()) { return QStringList(); } StringSet values; foreach(EntryPtr entry, m_entries) { values.add(FieldFormat::splitValue(entry->field(name_))); } // end entry loop - return values.toList(); + return values.values(); } Tellico::Data::FieldPtr Collection::fieldByName(const QString& name_) const { return FieldPtr(m_fieldByName.value(name_)); } Tellico::Data::FieldPtr Collection::fieldByTitle(const QString& title_) const { return FieldPtr(m_fieldByTitle.value(title_)); } bool Collection::hasField(const QString& name_) const { return m_fieldByName.contains(name_); } bool Collection::isAllowed(const QString& field_, const QString& value_) const { // empty string is always allowed if(value_.isEmpty()) { return true; } // find the field with a name of 'key_' FieldPtr field = fieldByName(field_); // if the type is not multiple choice or if value_ is allowed, return true if(field && (field->type() != Field::Choice || field->allowed().contains(value_))) { return true; } return false; } Tellico::Data::EntryGroupDict* Collection::entryGroupDictByName(const QString& name_) { // myDebug() << name_; m_lastGroupField = name_; // keep track, even if it's invalid if(name_.isEmpty() || !m_entryGroupDicts.contains(name_) || m_entries.isEmpty()) { return nullptr; } EntryGroupDict* dict = m_entryGroupDicts.value(name_); if(dict && dict->isEmpty()) { const bool b = signalsBlocked(); // block signals so all the group created/modified signals don't fire blockSignals(true); populateDict(dict, name_, m_entries); blockSignals(b); } return dict; } void Collection::populateDict(Tellico::Data::EntryGroupDict* dict_, const QString& fieldName_, const Tellico::Data::EntryList& entries_) { // myDebug() << fieldName_; Q_ASSERT(dict_); const bool isBool = hasField(fieldName_) && fieldByName(fieldName_)->type() == Field::Bool; QSet modifiedGroups; foreach(EntryPtr entry, entries_) { const QStringList groups = entryGroupNamesByField(entry, fieldName_); foreach(QString groupTitle, groups) { // krazy:exclude=foreach // find the group for this group name // bool fields use the field title if(isBool && !groupTitle.isEmpty()) { groupTitle = fieldTitleByName(fieldName_); } EntryGroup* group = dict_->value(groupTitle); // if the group doesn't exist, create it if(!group) { group = new EntryGroup(groupTitle, fieldName_); dict_->insert(groupTitle, group); } else if(group->isEmpty()) { // if it's empty, then it was previously added to the vector of groups to delete // remove it from that vector now that we're adding to it m_groupsToDelete.removeOne(group); } if(entry->addToGroup(group)) { modifiedGroups.insert(group); } } // end group loop } // end entry loop if(!modifiedGroups.isEmpty()) { - emit signalGroupsModified(CollPtr(this), modifiedGroups.toList()); + emit signalGroupsModified(CollPtr(this), modifiedGroups.values()); } } void Collection::populateCurrentDicts(const Tellico::Data::EntryList& entries_, const QStringList& fields_) { if(m_entryGroupDicts.isEmpty()) { return; } // special case when adding an entry to a new empty collection // there are no existing non-empty groups bool allEmpty = true; // iterate over all the possible groupDicts // for each dict, get the value of that field for the entry // if multiple values are allowed, split the value and then insert the // entry pointer into the dict for each value QHash::const_iterator dictIt = m_entryGroupDicts.constBegin(); for( ; dictIt != m_entryGroupDicts.constEnd(); ++dictIt) { // skip dicts for fields not in the modified list if(!fields_.contains(dictIt.key())) { continue; } // only populate if it's not empty, since they are // populated on demand if(!dictIt.value()->isEmpty()) { populateDict(dictIt.value(), dictIt.key(), entries_); allEmpty = false; } } if(allEmpty) { // myDebug() << "all collection dicts are empty"; // still need to populate the current group dict EntryGroupDict* dict = m_entryGroupDicts.value(m_lastGroupField); if(dict) { populateDict(dict, m_lastGroupField, entries_); } } } // return a string list for all the groups that the entry belongs to // for a given field. Normally, this would just be splitting the entry's value // for the field, but if the field name is the people pseudo-group, then it gets // a bit more complicated QStringList Collection::entryGroupNamesByField(Tellico::Data::EntryPtr entry_, const QString& fieldName_) { if(fieldName_ != s_peopleGroupName) { return entry_->groupNamesByFieldName(fieldName_); } // the empty group is only returned if the entry has an empty list for every people field bool allEmpty = true; StringSet values; foreach(FieldPtr field, m_peopleFields) { const QStringList groups = entry_->groupNamesByFieldName(field->name()); if(allEmpty && (groups.count() != 1 || !groups.at(0).isEmpty())) { allEmpty = false; } values.add(groups); } if(!allEmpty) { // we don't want the empty string values.remove(QString()); } - return values.toList(); + return values.values(); } void Collection::invalidateGroups() { foreach(EntryGroupDict* dict, m_entryGroupDicts) { qDeleteAll(*dict); dict->clear(); // don't delete the dict, just clear it } // populateDicts() will make signals that the group view is connected to, block those blockSignals(true); foreach(EntryPtr entry, m_entries) { entry->invalidateFormattedFieldValue(); entry->clearGroups(); } blockSignals(false); } Tellico::Data::EntryPtr Collection::entryById(Data::ID id_) { return EntryPtr(m_entryById.value(id_)); } void Collection::addBorrower(Tellico::Data::BorrowerPtr borrower_) { if(!borrower_) { return; } // check against existing borrower uid BorrowerPtr existingBorrower; foreach(BorrowerPtr bor, m_borrowers) { if(bor->uid() == borrower_->uid()) { existingBorrower = bor; break; } } if(!existingBorrower) { m_borrowers.append(borrower_); } else if(existingBorrower != borrower_) { // need to merge loans QHash existingLoans; foreach(LoanPtr loan, existingBorrower->loans()) { existingLoans.insert(loan->uid(), loan); } foreach(LoanPtr loan, borrower_->loans()) { if(!existingLoans.contains(loan->uid())) { existingBorrower->addLoan(loan); } } } } void Collection::addFilter(Tellico::FilterPtr filter_) { if(!filter_) { return; } m_filters.append(filter_); } bool Collection::removeFilter(Tellico::FilterPtr filter_) { if(!filter_) { return false; } return m_filters.removeAll(filter_) > 0; } void Collection::clear() { // since the collection holds a pointer to each entry and each entry // hold a pointer to the collection, and they're both sharedptrs, // neither will ever get deleted, unless the collection removes // all held pointers, specifically to entries m_fields.clear(); m_peopleFields.clear(); m_imageFields.clear(); m_fieldCategories.clear(); m_fieldByName.clear(); m_fieldByTitle.clear(); m_defaultGroupField.clear(); m_entries.clear(); m_entryById.clear(); foreach(EntryGroupDict* dict, m_entryGroupDicts) { qDeleteAll(*dict); } qDeleteAll(m_entryGroupDicts); m_entryGroupDicts.clear(); m_entryGroups.clear(); m_groupsToDelete.clear(); m_filters.clear(); m_borrowers.clear(); } void Collection::cleanGroups() { foreach(EntryGroup* group, m_groupsToDelete) { EntryGroupDict* dict = entryGroupDictByName(group->fieldName()); if(!dict) { continue; } EntryGroup* groupToDelete = dict->take(group->groupName()); delete groupToDelete; } m_groupsToDelete.clear(); } QString Collection::prepareText(const QString& text_) const { return text_; } int Collection::sameEntry(Tellico::Data::EntryPtr entry1_, Tellico::Data::EntryPtr entry2_) const { if(!entry1_ || !entry2_) { return 0; } // used to just return 0, but we really want a default generic implementation // that specific collections can override. int res = 0; // start with twice the title score // and since the minimum is > 10, then need more than just a perfect title match res += EntryComparison::MATCH_WEIGHT_MED*EntryComparison::score(entry1_, entry2_, QStringLiteral("title"), this); // then add score for each field foreach(FieldPtr field, entry1_->collection()->fields()) { res += EntryComparison::MATCH_WEIGHT_LOW*EntryComparison::score(entry1_, entry2_, field->name(), this); if(res >= EntryComparison::ENTRY_PERFECT_MATCH) return res; } return res; } Tellico::Data::ID Collection::getID() { static ID id = 0; return ++id; } Data::FieldPtr Collection::primaryImageField() const { return m_imageFields.isEmpty() ? Data::FieldPtr() : fieldByName(m_imageFields.front()->name()); } diff --git a/src/collections/bibtexcollection.cpp b/src/collections/bibtexcollection.cpp index e2bba87a..895cd24e 100644 --- a/src/collections/bibtexcollection.cpp +++ b/src/collections/bibtexcollection.cpp @@ -1,526 +1,526 @@ /*************************************************************************** Copyright (C) 2003-2009 Robby Stephenson ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * * * ***************************************************************************/ #include "bibtexcollection.h" #include "../entrycomparison.h" #include "../utils/bibtexhandler.h" #include "../fieldformat.h" #include "../tellico_debug.h" #include #include using namespace Tellico; using Tellico::Data::BibtexCollection; namespace { static const char* bibtex_general = I18N_NOOP("General"); static const char* bibtex_publishing = I18N_NOOP("Publishing"); static const char* bibtex_misc = I18N_NOOP("Miscellaneous"); } BibtexCollection::BibtexCollection(bool addDefaultFields_, const QString& title_) : Collection(title_.isEmpty() ? i18n("Bibliography") : title_) { setDefaultGroupField(QStringLiteral("author")); if(addDefaultFields_) { addFields(defaultFields()); } // Bibtex has some default macros for the months addMacro(QStringLiteral("jan"), QString()); addMacro(QStringLiteral("feb"), QString()); addMacro(QStringLiteral("mar"), QString()); addMacro(QStringLiteral("apr"), QString()); addMacro(QStringLiteral("may"), QString()); addMacro(QStringLiteral("jun"), QString()); addMacro(QStringLiteral("jul"), QString()); addMacro(QStringLiteral("aug"), QString()); addMacro(QStringLiteral("sep"), QString()); addMacro(QStringLiteral("oct"), QString()); addMacro(QStringLiteral("nov"), QString()); addMacro(QStringLiteral("dec"), QString()); } Tellico::Data::FieldList BibtexCollection::defaultFields() { FieldList list; FieldPtr field; const QString bibtex = QStringLiteral("bibtex"); /******************* General ****************************/ field = Field::createDefaultField(Field::TitleField); field->setProperty(bibtex, QStringLiteral("title")); list.append(field); QStringList types; types << QStringLiteral("article") << QStringLiteral("book") << QStringLiteral("booklet") << QStringLiteral("inbook") << QStringLiteral("incollection") << QStringLiteral("inproceedings") << QStringLiteral("manual") << QStringLiteral("mastersthesis") << QStringLiteral("misc") << QStringLiteral("phdthesis") << QStringLiteral("proceedings") << QStringLiteral("techreport") << QStringLiteral("unpublished") << QStringLiteral("periodical") << QStringLiteral("conference"); field = new Field(QStringLiteral("entry-type"), i18n("Entry Type"), types); field->setProperty(bibtex, QStringLiteral("entry-type")); field->setCategory(i18n(bibtex_general)); field->setFlags(Field::AllowGrouped | Field::NoDelete); field->setDescription(i18n("These entry types are specific to bibtex. See the bibtex documentation.")); list.append(field); field = new Field(QStringLiteral("author"), i18n("Author")); field->setProperty(bibtex, QStringLiteral("author")); field->setCategory(i18n(bibtex_general)); field->setFlags(Field::AllowCompletion | Field::AllowMultiple | Field::AllowGrouped); field->setFormatType(FieldFormat::FormatName); list.append(field); field = new Field(QStringLiteral("bibtex-key"), i18n("Bibtex Key")); field->setProperty(bibtex, QStringLiteral("key")); field->setCategory(i18n("General")); field->setFlags(Field::NoDelete); list.append(field); field = new Field(QStringLiteral("booktitle"), i18n("Book Title")); field->setProperty(bibtex, QStringLiteral("booktitle")); field->setCategory(i18n(bibtex_general)); field->setFormatType(FieldFormat::FormatTitle); list.append(field); field = new Field(QStringLiteral("editor"), i18n("Editor")); field->setProperty(bibtex, QStringLiteral("editor")); field->setCategory(i18n(bibtex_general)); field->setFlags(Field::AllowCompletion | Field::AllowMultiple | Field::AllowGrouped); field->setFormatType(FieldFormat::FormatName); list.append(field); field = new Field(QStringLiteral("organization"), i18n("Organization")); field->setProperty(bibtex, QStringLiteral("organization")); field->setCategory(i18n(bibtex_general)); field->setFlags(Field::AllowCompletion | Field::AllowGrouped); field->setFormatType(FieldFormat::FormatPlain); list.append(field); // field = new Field(QLatin1String("institution"), i18n("Institution")); // field->setProperty(QLatin1String("bibtex"), QLatin1String("institution")); // field->setCategory(i18n(bibtex_general)); // field->setFlags(Field::AllowDelete); // field->setFormatType(FieldFormat::FormatTitle); // list.append(field); /******************* Publishing ****************************/ field = new Field(QStringLiteral("publisher"), i18n("Publisher")); field->setProperty(bibtex, QStringLiteral("publisher")); field->setCategory(i18n(bibtex_publishing)); field->setFlags(Field::AllowCompletion | Field::AllowGrouped); field->setFormatType(FieldFormat::FormatPlain); list.append(field); field = new Field(QStringLiteral("address"), i18n("Address")); field->setProperty(bibtex, QStringLiteral("address")); field->setCategory(i18n(bibtex_publishing)); field->setFlags(Field::AllowCompletion | Field::AllowGrouped); list.append(field); field = new Field(QStringLiteral("edition"), i18n("Edition")); field->setProperty(bibtex, QStringLiteral("edition")); field->setCategory(i18n(bibtex_publishing)); field->setFlags(Field::AllowCompletion); list.append(field); // don't make it a number, it could have latex processing commands in it field = new Field(QStringLiteral("pages"), i18n("Pages")); field->setProperty(bibtex, QStringLiteral("pages")); field->setCategory(i18n(bibtex_publishing)); list.append(field); field = new Field(QStringLiteral("year"), i18n("Year"), Field::Number); field->setProperty(bibtex, QStringLiteral("year")); field->setCategory(i18n(bibtex_publishing)); field->setFlags(Field::AllowGrouped); list.append(field); field = Field::createDefaultField(Field::IsbnField); field->setProperty(bibtex, QStringLiteral("isbn")); field->setCategory(i18n(bibtex_publishing)); list.append(field); field = new Field(QStringLiteral("journal"), i18n("Journal")); field->setProperty(bibtex, QStringLiteral("journal")); field->setCategory(i18n(bibtex_publishing)); field->setFlags(Field::AllowCompletion | Field::AllowGrouped); field->setFormatType(FieldFormat::FormatPlain); list.append(field); field = new Field(QStringLiteral("doi"), i18n("DOI")); field->setProperty(bibtex, QStringLiteral("doi")); field->setCategory(i18n(bibtex_publishing)); field->setDescription(i18n("Digital Object Identifier")); list.append(field); // could make this a string list, but since bibtex import could have funky values // keep it an editbox field = new Field(QStringLiteral("month"), i18n("Month")); field->setProperty(bibtex, QStringLiteral("month")); field->setCategory(i18n(bibtex_publishing)); field->setFlags(Field::AllowCompletion); list.append(field); field = new Field(QStringLiteral("number"), i18n("Number"), Field::Number); field->setProperty(bibtex, QStringLiteral("number")); field->setCategory(i18n(bibtex_publishing)); list.append(field); field = new Field(QStringLiteral("howpublished"), i18n("How Published")); field->setProperty(bibtex, QStringLiteral("howpublished")); field->setCategory(i18n(bibtex_publishing)); list.append(field); // field = new Field(QLatin1String("school"), i18n("School")); // field->setProperty(QLatin1String("bibtex"), QLatin1String("school")); // field->setCategory(i18n(bibtex_publishing)); // field->setFlags(Field::AllowCompletion | Field::AllowGrouped); // list.append(field); /******************* Classification ****************************/ field = new Field(QStringLiteral("chapter"), i18n("Chapter"), Field::Number); field->setProperty(bibtex, QStringLiteral("chapter")); field->setCategory(i18n(bibtex_misc)); list.append(field); field = new Field(QStringLiteral("series"), i18n("Series")); field->setProperty(bibtex, QStringLiteral("series")); field->setCategory(i18n(bibtex_misc)); field->setFlags(Field::AllowCompletion | Field::AllowGrouped); field->setFormatType(FieldFormat::FormatTitle); list.append(field); field = new Field(QStringLiteral("volume"), i18nc("A number field in a bibliography", "Volume"), Field::Number); field->setProperty(bibtex, QStringLiteral("volume")); field->setCategory(i18n(bibtex_misc)); list.append(field); field = new Field(QStringLiteral("crossref"), i18n("Cross-Reference")); field->setProperty(bibtex, QStringLiteral("crossref")); field->setCategory(i18n(bibtex_misc)); list.append(field); // field = new Field(QLatin1String("annote"), i18n("Annotation")); // field->setProperty(QLatin1String("bibtex"), QLatin1String("annote")); // field->setCategory(i18n(bibtex_misc)); // list.append(field); field = new Field(QStringLiteral("keyword"), i18n("Keywords")); field->setProperty(bibtex, QStringLiteral("keywords")); field->setCategory(i18n(bibtex_misc)); field->setFlags(Field::AllowCompletion | Field::AllowMultiple | Field::AllowGrouped); list.append(field); field = new Field(QStringLiteral("url"), i18n("URL"), Field::URL); field->setProperty(bibtex, QStringLiteral("url")); field->setCategory(i18n(bibtex_misc)); list.append(field); field = new Field(QStringLiteral("abstract"), i18n("Abstract"), Field::Para); field->setProperty(bibtex, QStringLiteral("abstract")); list.append(field); field = new Field(QStringLiteral("note"), i18n("Notes"), Field::Para); field->setProperty(bibtex, QStringLiteral("note")); list.append(field); field = Field::createDefaultField(Field::IDField); field->setCategory(i18n(bibtex_misc)); list.append(field); field = Field::createDefaultField(Field::CreatedDateField); field->setCategory(i18n(bibtex_misc)); list.append(field); field = Field::createDefaultField(Field::ModifiedDateField); field->setCategory(i18n(bibtex_misc)); list.append(field); return list; } bool BibtexCollection::addField(Tellico::Data::FieldPtr field_) { if(!field_) { return false; } bool success = Collection::addField(field_); if(success) { const QString bibtex = field_->property(QStringLiteral("bibtex")); if(!bibtex.isEmpty()) { m_bibtexFieldDict.insert(bibtex, field_.data()); } } return success; } bool BibtexCollection::modifyField(Tellico::Data::FieldPtr newField_) { if(!newField_) { return false; } // myDebug(); const QString bibtex = QStringLiteral("bibtex"); bool success = Collection::modifyField(newField_); FieldPtr oldField = fieldByName(newField_->name()); const QString oldBibtex = oldField->property(bibtex); const QString newBibtex = newField_->property(bibtex); if(!oldBibtex.isEmpty()) { success &= (m_bibtexFieldDict.remove(oldBibtex) != 0); } if(!newBibtex.isEmpty()) { oldField->setProperty(bibtex, newBibtex); m_bibtexFieldDict.insert(newBibtex, oldField.data()); } return success; } bool BibtexCollection::removeField(Tellico::Data::FieldPtr field_, bool force_) { if(!field_) { return false; } // myDebug(); bool success = true; const QString bibtex = field_->property(QStringLiteral("bibtex")); if(!bibtex.isEmpty()) { success &= (m_bibtexFieldDict.remove(bibtex) != 0); } return success && Collection::removeField(field_, force_); } bool BibtexCollection::removeField(const QString& name_, bool force_) { return removeField(fieldByName(name_), force_); } Tellico::Data::FieldPtr BibtexCollection::fieldByBibtexName(const QString& bibtex_) const { return FieldPtr(m_bibtexFieldDict.contains(bibtex_) ? m_bibtexFieldDict.value(bibtex_) : nullptr); } Tellico::Data::EntryPtr BibtexCollection::entryByBibtexKey(const QString& key_) const { EntryPtr entry; // we do assume unique keys foreach(EntryPtr e, entries()) { if(e->field(QStringLiteral("bibtex-key")) == key_) { entry = e; break; } } return entry; } QString BibtexCollection::prepareText(const QString& text_) const { QString text = text_; BibtexHandler::cleanText(text); return text; } // same as BookCollection::sameEntry() int BibtexCollection::sameEntry(Tellico::Data::EntryPtr entry1_, Tellico::Data::EntryPtr entry2_) const { // equal identifiers are easy, give it a weight of 100 if(EntryComparison::score(entry1_, entry2_, QStringLiteral("isbn"), this) > 0 || EntryComparison::score(entry1_, entry2_, QStringLiteral("lccn"), this) > 0 || EntryComparison::score(entry1_, entry2_, QStringLiteral("doi"), this) > 0 || EntryComparison::score(entry1_, entry2_, QStringLiteral("pmid"), this) > 0 || EntryComparison::score(entry1_, entry2_, QStringLiteral("arxiv"), this) > 0) { return 100; // good match } int res = 3*EntryComparison::score(entry1_, entry2_, QStringLiteral("title"), this); // if(res == 0) { // myDebug() << "different titles for " << entry1_->title() << " vs. " // << entry2_->title(); // } res += EntryComparison::score(entry1_, entry2_, QStringLiteral("author"), this); res += EntryComparison::score(entry1_, entry2_, QStringLiteral("cr_year"), this); res += EntryComparison::score(entry1_, entry2_, QStringLiteral("pub_year"), this); res += EntryComparison::score(entry1_, entry2_, QStringLiteral("binding"), this); return res; } // static Tellico::Data::CollPtr BibtexCollection::convertBookCollection(Tellico::Data::CollPtr coll_) { const QString bibtex = QStringLiteral("bibtex"); BibtexCollection* coll = new BibtexCollection(false, coll_->title()); CollPtr collPtr(coll); FieldList fields = coll_->fields(); foreach(FieldPtr fIt, fields) { FieldPtr field(new Field(*fIt)); // if it already has a bibtex property, skip it if(!field->property(bibtex).isEmpty()) { coll->addField(field); continue; } // be sure to set bibtex property before adding it though QString name = field->name(); // this first group has bibtex field names the same as their own field name if(name == QLatin1String("title") || name == QLatin1String("author") || name == QLatin1String("editor") || name == QLatin1String("edition") || name == QLatin1String("publisher") || name == QLatin1String("isbn") || name == QLatin1String("lccn") || name == QLatin1String("url") || name == QLatin1String("language") || name == QLatin1String("pages") || name == QLatin1String("series")) { field->setProperty(bibtex, name); } else if(name == QLatin1String("series_num")) { field->setProperty(bibtex, QStringLiteral("number")); } else if(name == QLatin1String("pur_price")) { field->setProperty(bibtex, QStringLiteral("price")); } else if(name == QLatin1String("cr_year")) { field->setProperty(bibtex, QStringLiteral("year")); } else if(name == QLatin1String("bibtex-id")) { field->setProperty(bibtex, QStringLiteral("key")); } else if(name == QLatin1String("keyword")) { field->setProperty(bibtex, QStringLiteral("keywords")); } else if(name == QLatin1String("comments")) { field->setProperty(bibtex, QStringLiteral("note")); } coll->addField(field); } // also need to add required fields, those with NoDelete set foreach(FieldPtr defaultField, coll->defaultFields()) { if(!coll->hasField(defaultField->name()) && defaultField->hasFlag(Field::NoDelete)) { // but don't add a Bibtex Key if there's already a bibtex-id if(defaultField->property(bibtex) != QLatin1String("key") || !coll->hasField(QStringLiteral("bibtex-id"))) { coll->addField(defaultField); } } } // set the entry-type to book FieldPtr field = coll->fieldByBibtexName(QStringLiteral("entry-type")); QString entryTypeName; if(field) { entryTypeName = field->name(); } else { myWarning() << "there must be an entry type field"; } EntryList newEntries; foreach(EntryPtr entry, coll_->entries()) { EntryPtr newEntry(new Entry(*entry)); newEntry->setCollection(collPtr); if(!entryTypeName.isEmpty()) { newEntry->setField(entryTypeName, QStringLiteral("book")); } newEntries.append(newEntry); } coll->addEntries(newEntries); return collPtr; } bool BibtexCollection::setFieldValue(Data::EntryPtr entry_, const QString& bibtexField_, const QString& value_, Data::CollPtr existingColl_) { Q_ASSERT(entry_->collection()->type() == Collection::Bibtex); BibtexCollection* c = static_cast(entry_->collection().data()); FieldPtr field = c->fieldByBibtexName(bibtexField_); // special-case: "keyword" and "keywords" should be the same field. if(!field && bibtexField_ == QLatin1String("keyword")) { field = c->fieldByBibtexName(QStringLiteral("keywords")); } if(!field) { // it was the case that the default bibliography did not have a bibtex property for keywords // so a "keywords" field would get created in the imported collection // but the existing collection had a field "keyword" so the values would not get imported // here, check to see if the current collection has a field with the same bibtex name and // use it instead of creating a new one BibtexCollection* existingColl = dynamic_cast(existingColl_.data()); FieldPtr existingField; if(existingColl && existingColl->type() == Collection::Bibtex) { existingField = existingColl->fieldByBibtexName(bibtexField_); } if(existingField) { field = new Field(*existingField); } else if(value_.length() < 100) { // arbitrarily say if the value has more than 100 chars, then it's a paragraph QString vlower = value_.toLower(); // special case, try to detect URLs if(bibtexField_ == QLatin1String("url") || vlower.startsWith(QLatin1String("http")) // may also be https || vlower.startsWith(QLatin1String("ftp:/")) || vlower.startsWith(QLatin1String("file:/")) || vlower.startsWith(QLatin1String("/"))) { // assume this indicates a local path myDebug() << "creating a URL field for" << bibtexField_; field = new Field(bibtexField_, KStringHandler::capwords(bibtexField_), Field::URL); } else { myDebug() << "creating a LINE field for" << bibtexField_; field = new Field(bibtexField_, KStringHandler::capwords(bibtexField_), Field::Line); } field->setCategory(i18n("Unknown")); } else { myDebug() << "creating a PARA field for" << bibtexField_; field = new Field(bibtexField_, KStringHandler::capwords(bibtexField_), Field::Para); } field->setProperty(QStringLiteral("bibtex"), bibtexField_); c->addField(field); } // special case keywords, replace commas with semi-colons so they get separated QString value = value_; Q_ASSERT(field); if(bibtexField_.startsWith(QLatin1String("keyword"))) { value.replace(QRegExp(QLatin1String("\\s*,\\s*")), FieldFormat::delimiterString()); // special case refbase bibtex export, with multiple keywords fields QString oValue = entry_->field(field); if(!oValue.isEmpty()) { value = oValue + FieldFormat::delimiterString() + value; } // special case for tilde, since it's a non-breaking space in LateX // replace it EXCEPT for URL or DOI fields } else if(bibtexField_ != QLatin1String("doi") && field->type() != Field::URL) { value.replace(QLatin1Char('~'), QChar(0xA0)); } else if(field->type() == Field::URL || bibtexField_ == QLatin1String("url")) { // special case for url package if(value.startsWith(QLatin1String("\\url{")) && value.endsWith(QLatin1Char('}'))) { value.remove(0, 5).chop(1); } } return entry_->setField(field, value); } Tellico::Data::EntryList BibtexCollection::duplicateBibtexKeys() const { QSet dupes; QHash keyHash; const QString keyField = QStringLiteral("bibtex-key"); QString keyValue; foreach(EntryPtr entry, entries()) { keyValue = entry->field(keyField); if(keyHash.contains(keyValue)) { dupes << keyHash.value(keyValue) << entry; } else { keyHash.insert(keyValue, entry); } } - return dupes.toList(); + return dupes.values(); } diff --git a/src/document.cpp b/src/document.cpp index 28052ad1..74cf6df6 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -1,855 +1,855 @@ /*************************************************************************** Copyright (C) 2001-2009 Robby Stephenson ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * * * ***************************************************************************/ #include "document.h" #include "collectionfactory.h" #include "translators/tellicoimporter.h" #include "translators/tellicozipexporter.h" #include "translators/tellicoxmlexporter.h" #include "collection.h" #include "core/filehandler.h" #include "borrower.h" #include "fieldformat.h" #include "core/tellico_strings.h" #include "images/imagefactory.h" #include "images/imagedirectory.h" #include "images/image.h" #include "images/imageinfo.h" #include "utils/stringset.h" #include "progressmanager.h" #include "config/tellico_config.h" #include "entrycomparison.h" #include "utils/guiproxy.h" #include "tellico_debug.h" #include #include #include #include #include #include using namespace Tellico; using Tellico::Data::Document; Document* Document::s_self = nullptr; Document::Document() : QObject(), m_coll(nullptr), m_isModified(false), m_loadAllImages(false), m_validFile(false), m_importer(nullptr), m_cancelImageWriting(true), m_fileFormat(Import::TellicoImporter::Unknown) { m_allImagesOnDisk = Config::imageLocation() != Config::ImagesInFile; newDocument(Collection::Book); } Document::~Document() { delete m_importer; m_importer = nullptr; } Tellico::Data::CollPtr Document::collection() const { return m_coll; } void Document::setURL(const QUrl& url_) { m_url = url_; if(m_url.fileName() != i18n(Tellico::untitledFilename)) { ImageFactory::setLocalDirectory(m_url); EntryComparison::setDocumentUrl(m_url); } } void Document::setModified(bool modified_) { if(modified_ != m_isModified) { m_isModified = modified_; emit signalModified(m_isModified); } } void Document::slotSetModified() { setModified(true); } /** * Since QUndoStack emits cleanChanged(), the behavior is opposite * the document modified flag */ void Document::slotSetClean(bool clean_) { setModified(!clean_); } bool Document::newDocument(int type_) { if(m_importer) { m_importer->deleteLater(); m_importer = nullptr; } deleteContents(); m_coll = CollectionFactory::collection(type_, true); m_coll->setTrackGroups(true); emit signalCollectionAdded(m_coll); emit signalCollectionImagesLoaded(m_coll); setModified(false); QUrl url = QUrl::fromLocalFile(i18n(Tellico::untitledFilename)); setURL(url); m_validFile = false; m_fileFormat = Import::TellicoImporter::Unknown; return true; } bool Document::openDocument(const QUrl& url_) { MARK; m_loadAllImages = false; // delayed image loading only works for local files if(!url_.isLocalFile()) { m_loadAllImages = true; } if(m_importer) { m_importer->deleteLater(); } m_importer = new Import::TellicoImporter(url_, m_loadAllImages); ProgressItem& item = ProgressManager::self()->newProgressItem(m_importer, m_importer->progressLabel(), true); connect(m_importer, &Import::Importer::signalTotalSteps, ProgressManager::self(), &ProgressManager::setTotalSteps); connect(m_importer, &Import::Importer::signalProgress, ProgressManager::self(), &ProgressManager::setProgress); connect(&item, &ProgressItem::signalCancelled, m_importer, &Import::Importer::slotCancel); ProgressItem::Done done(m_importer); CollPtr coll = m_importer->collection(); if(!m_importer) { myDebug() << "The importer was deleted out from under us"; return false; } // delayed image loading only works for zip files // format is only known AFTER collection() is called m_fileFormat = m_importer->format(); m_allImagesOnDisk = !m_importer->hasImages(); if(!m_importer->hasImages() || m_fileFormat != Import::TellicoImporter::Zip) { m_loadAllImages = true; } ImageFactory::setZipArchive(m_importer->takeImages()); if(!coll) { // myDebug() << "returning false"; GUI::Proxy::sorry(m_importer->statusMessage()); m_validFile = false; return false; } deleteContents(); m_coll = coll; m_coll->setTrackGroups(true); setURL(url_); m_validFile = true; emit signalCollectionAdded(m_coll); // m_importer might have been deleted? setModified(m_importer && m_importer->modifiedOriginal()); // if(pruneImages()) { // slotSetModified(true); // } if(m_importer && m_importer->hasImages()) { m_cancelImageWriting = false; QTimer::singleShot(500, this, &Document::slotLoadAllImages); } else { emit signalCollectionImagesLoaded(m_coll); if(m_importer) { m_importer->deleteLater(); m_importer = nullptr; } } return true; } bool Document::saveDocument(const QUrl& url_, bool force_) { // FileHandler::queryExists calls FileHandler::writeBackupFile // so the only reason to check queryExists() is if the url to write to is different than the current one if(url_ == m_url) { if(!FileHandler::writeBackupFile(url_)) { return false; } } else { if(!force_ && !FileHandler::queryExists(url_)) { return false; } } // in case we're still loading images, give that a chance to cancel m_cancelImageWriting = true; qApp->processEvents(); ProgressItem& item = ProgressManager::self()->newProgressItem(this, i18n("Saving file..."), false); ProgressItem::Done done(this); // will always save as zip file, no matter if has images or not int imageLocation = Config::imageLocation(); bool includeImages = imageLocation == Config::ImagesInFile; int totalSteps; // write all images to disk cache if needed // have to do this before executing exporter in case // the user changed the imageInFile setting from Yes to No, in which // case saving will overwrite the old file that has the images in it! if(includeImages) { totalSteps = 10; item.setTotalSteps(totalSteps); // since TellicoZipExporter uses 100 steps, then it will get 100/110 of the total progress } else { totalSteps = 100; item.setTotalSteps(totalSteps); m_cancelImageWriting = false; writeAllImages(imageLocation == Config::ImagesInAppDir ? ImageFactory::DataDir : ImageFactory::LocalDir, url_); } QScopedPointer exporter; if(m_fileFormat == Import::TellicoImporter::XML) { exporter.reset(new Export::TellicoXMLExporter(m_coll)); static_cast(exporter.data())->setIncludeImages(includeImages); } else { exporter.reset(new Export::TellicoZipExporter(m_coll)); static_cast(exporter.data())->setIncludeImages(includeImages); } item.setProgress(int(0.8*totalSteps)); exporter->setEntries(m_coll->entries()); exporter->setURL(url_); // since we already asked about overwriting the file, force the save long opt = exporter->options() | Export::ExportForce | Export::ExportComplete | Export::ExportProgress; // only write the image sizes if they're known already opt &= ~Export::ExportImageSize; exporter->setOptions(opt); const bool success = exporter->exec(); item.setProgress(int(0.9*totalSteps)); if(success) { setURL(url_); // if successful, doc is no longer modified setModified(false); } else { myDebug() << "Document::saveDocument() - not successful saving to" << url_.url(); } return success; } bool Document::closeDocument() { if(m_importer) { m_importer->deleteLater(); m_importer = nullptr; } deleteContents(); return true; } void Document::deleteContents() { if(m_coll) { emit signalCollectionDeleted(m_coll); } // don't delete the m_importer here, bad things will happen // since the collection holds a pointer to each entry and each entry // hold a pointer to the collection, and they're both sharedptrs, // neither will ever get deleted, unless the entries are removed from the collection if(m_coll) { m_coll->clear(); } m_coll = nullptr; // old collection gets deleted as refcount goes to 0 m_cancelImageWriting = true; } void Document::appendCollection(Tellico::Data::CollPtr coll_) { appendCollection(m_coll, coll_); } void Document::appendCollection(Tellico::Data::CollPtr coll1_, Tellico::Data::CollPtr coll2_) { if(!coll1_ || !coll2_) { return; } coll1_->blockSignals(true); foreach(FieldPtr field, coll2_->fields()) { coll1_->mergeField(field); } Data::EntryList newEntries; foreach(EntryPtr entry, coll2_->entries()) { Data::EntryPtr newEntry(new Data::Entry(*entry)); newEntry->setCollection(coll1_); newEntries << newEntry; } coll1_->addEntries(newEntries); // TODO: merge filters and loans coll1_->blockSignals(false); } Tellico::Data::MergePair Document::mergeCollection(Tellico::Data::CollPtr coll_) { return mergeCollection(m_coll, coll_); } Tellico::Data::MergePair Document::mergeCollection(Tellico::Data::CollPtr coll1_, Tellico::Data::CollPtr coll2_) { MergePair pair; if(!coll1_ || !coll2_) { return pair; } coll1_->blockSignals(true); Data::FieldList fields = coll2_->fields(); foreach(FieldPtr field, fields) { coll1_->mergeField(field); } EntryList currEntries = coll1_->entries(); EntryList newEntries = coll2_->entries(); std::sort(currEntries.begin(), currEntries.end(), Data::EntryCmp(QStringLiteral("title"))); std::sort(newEntries.begin(), newEntries.end(), Data::EntryCmp(QStringLiteral("title"))); const int currTotal = currEntries.count(); int lastMatchId = 0; bool checkSameId = false; // if the matching entries have the same id, then check that first for later comparisons foreach(EntryPtr newEntry, newEntries) { int bestMatch = 0; Data::EntryPtr matchEntry, currEntry; // first, if we're checking against same ID if(checkSameId) { currEntry = coll1_->entryById(newEntry->id()); if(currEntry && coll1_->sameEntry(currEntry, newEntry) >= EntryComparison::ENTRY_PERFECT_MATCH) { // only have to compare against perfect match matchEntry = currEntry; } } if(!matchEntry) { // alternative is to loop over them all for(int i = 0; i < currTotal; ++i) { // since we're sorted by title, track the index of the previous match and start comparison there currEntry = currEntries.at((i+lastMatchId) % currTotal); const int match = coll1_->sameEntry(currEntry, newEntry); if(match >= EntryComparison::ENTRY_PERFECT_MATCH) { matchEntry = currEntry; lastMatchId = (i+lastMatchId) % currTotal; break; } else if(match >= EntryComparison::ENTRY_GOOD_MATCH && match > bestMatch) { bestMatch = match; matchEntry = currEntry; lastMatchId = (i+lastMatchId) % currTotal; // don't break, keep looking for better one } } } if(matchEntry) { checkSameId = checkSameId || (matchEntry->id() == newEntry->id()); mergeEntry(matchEntry, newEntry); } else { Data::EntryPtr e(new Data::Entry(*newEntry)); e->setCollection(coll1_); // keep track of which entries got added pair.first.append(e); } } coll1_->addEntries(pair.first); // TODO: merge filters and loans coll1_->blockSignals(false); return pair; } void Document::replaceCollection(Tellico::Data::CollPtr coll_) { if(!coll_) { return; } QUrl url = QUrl::fromLocalFile(i18n(Tellico::untitledFilename)); setURL(url); m_validFile = false; // the collection gets cleared by the CollectionCommand that called this function // no need to do it here m_coll = coll_; m_coll->setTrackGroups(true); m_cancelImageWriting = true; // CollectionCommand takes care of calling Controller signals } void Document::unAppendCollection(Tellico::Data::CollPtr coll_, Tellico::Data::FieldList origFields_) { if(!coll_) { return; } m_coll->blockSignals(true); StringSet origFieldNames; foreach(FieldPtr field, origFields_) { m_coll->modifyField(field); origFieldNames.add(field->name()); } EntryList entries = coll_->entries(); foreach(EntryPtr entry, entries) { // probably don't need to do this, but on the safe side... entry->setCollection(coll_); } m_coll->removeEntries(entries); // since Collection::removeField() iterates over all entries to reset the value of the field // don't removeField() until after removeEntry() is done FieldList currFields = m_coll->fields(); foreach(FieldPtr field, currFields) { if(!origFieldNames.has(field->name())) { m_coll->removeField(field); } } m_coll->blockSignals(false); } void Document::unMergeCollection(Tellico::Data::CollPtr coll_, Tellico::Data::FieldList origFields_, Tellico::Data::MergePair entryPair_) { if(!coll_) { return; } m_coll->blockSignals(true); QStringList origFieldNames; foreach(FieldPtr field, origFields_) { m_coll->modifyField(field); origFieldNames << field->name(); } // first item in pair are the entries added by the operation, remove them EntryList entries = entryPair_.first; m_coll->removeEntries(entries); // second item in pair are the entries which got modified by the original merge command const QString track = QStringLiteral("track"); PairVector trackChanges = entryPair_.second; // need to go through them in reverse since one entry may have been modified multiple times // first item in the pair is the entry pointer // second item is the old value of the track field for(int i = trackChanges.count()-1; i >= 0; --i) { trackChanges[i].first->setField(track, trackChanges[i].second); } // since Collection::removeField() iterates over all entries to reset the value of the field // don't removeField() until after removeEntry() is done FieldList currFields = m_coll->fields(); foreach(FieldPtr field, currFields) { if(origFieldNames.indexOf(field->name()) == -1) { m_coll->removeField(field); } } m_coll->blockSignals(false); } bool Document::isEmpty() const { //an empty doc may contain a collection, but no entries return (!m_coll || m_coll->entries().isEmpty()); } bool Document::loadAllImagesNow() const { // DEBUG_LINE; if(!m_coll || !m_validFile) { return false; } if(m_loadAllImages) { myDebug() << "Document::loadAllImagesNow() - all valid images should already be loaded!"; return false; } return Import::TellicoImporter::loadAllImages(m_url); } Tellico::Data::EntryList Document::filteredEntries(Tellico::FilterPtr filter_) const { Data::EntryList matches; Data::EntryList entries = m_coll->entries(); foreach(EntryPtr entry, entries) { if(filter_->matches(entry)) { matches.append(entry); } } return matches; } void Document::checkOutEntry(Tellico::Data::EntryPtr entry_) { if(!entry_) { return; } const QString loaned = QStringLiteral("loaned"); if(!m_coll->hasField(loaned)) { FieldPtr f(new Field(loaned, i18n("Loaned"), Field::Bool)); f->setFlags(Field::AllowGrouped); f->setCategory(i18n("Personal")); m_coll->addField(f); } entry_->setField(loaned, QStringLiteral("true")); EntryList vec; vec.append(entry_); m_coll->updateDicts(vec, QStringList() << loaned); } void Document::checkInEntry(Tellico::Data::EntryPtr entry_) { if(!entry_) { return; } const QString loaned = QStringLiteral("loaned"); if(!m_coll->hasField(loaned)) { return; } entry_->setField(loaned, QString()); m_coll->updateDicts(EntryList() << entry_, QStringList() << loaned); } void Document::renameCollection(const QString& newTitle_) { m_coll->setTitle(newTitle_); } // this only gets called when a zip file with images is opened // by loading every image, it gets pulled out of the zip file and // copied to disk. Then the zip file can be closed and not retained in memory void Document::slotLoadAllImages() { QString id; StringSet images; foreach(EntryPtr entry, m_coll->entries()) { foreach(FieldPtr field, m_coll->imageFields()) { id = entry->field(field); if(id.isEmpty() || images.has(id)) { continue; } // this is the early loading, so just by calling imageById() // the image gets sucked from the zip file and written to disk // by ImageFactory::imageById() // TODO:: does this need to check against images with link only? if(ImageFactory::imageById(id).isNull()) { myDebug() << "Null image for entry:" << entry->title() << id; } images.add(id); if(m_cancelImageWriting) { break; } } if(m_cancelImageWriting) { break; } // stay responsive, do this in the background qApp->processEvents(); } if(m_cancelImageWriting) { myLog() << "slotLoadAllImages() - cancel image writing"; } else { emit signalCollectionImagesLoaded(m_coll); } m_cancelImageWriting = false; if(m_importer) { m_importer->deleteLater(); m_importer = nullptr; } } // cacheDir_ is the location dir to write the images // localDir_ provide the new file location which is only needed if cacheDir == LocalDir void Document::writeAllImages(int cacheDir_, const QUrl& localDir_) { // images get 80 steps in saveDocument() const uint stepSize = 1 + qMax(1, m_coll->entryCount()/80); // add 1 since it could round off uint j = 1; ImageFactory::CacheDir cacheDir = static_cast(cacheDir_); QScopedPointer imgDir; if(cacheDir == ImageFactory::LocalDir) { imgDir.reset(new ImageDirectory(ImageFactory::localDirectory(localDir_))); } QString id; StringSet images; EntryList entries = m_coll->entries(); FieldList imageFields = m_coll->imageFields(); foreach(EntryPtr entry, entries) { foreach(FieldPtr field, imageFields) { id = entry->field(field); if(id.isEmpty() || images.has(id)) { continue; } images.add(id); if(ImageFactory::imageInfo(id).linkOnly) { continue; } // careful here, if we're writing to LocalDir, need to read from the old LocalDir and write to new bool success; if(cacheDir == ImageFactory::LocalDir) { success = ImageFactory::writeCachedImage(id, imgDir.data()); } else { success = ImageFactory::writeCachedImage(id, cacheDir); } if(!success) { myDebug() << "did not write image for entry title:" << entry->title(); } if(m_cancelImageWriting) { break; } } if(j%stepSize == 0) { ProgressManager::self()->setProgress(this, j/stepSize); qApp->processEvents(); } ++j; if(m_cancelImageWriting) { break; } } if(m_cancelImageWriting) { myDebug() << "Document::writeAllImages() - cancel image writing"; } m_cancelImageWriting = false; } bool Document::pruneImages() { bool found = false; QString id; StringSet images; Data::EntryList entries = m_coll->entries(); Data::FieldList imageFields = m_coll->imageFields(); foreach(EntryPtr entry, entries) { foreach(FieldPtr field, imageFields) { id = entry->field(field); if(id.isEmpty() || images.has(id)) { continue; } const Data::Image& img = ImageFactory::imageById(id); if(img.isNull()) { entry->setField(field, QString()); found = true; myDebug() << "removing null image for" << entry->title() << ":" << id; } else { images.add(id); } } } return found; } int Document::imageCount() const { if(!m_coll) { return 0; } StringSet images; FieldList fields = m_coll->imageFields(); EntryList entries = m_coll->entries(); foreach(FieldPtr field, fields) { foreach(EntryPtr entry, entries) { images.add(entry->field(field->name())); } } return images.count(); } void Document::removeImagesNotInCollection(Tellico::Data::EntryList entries_, Tellico::Data::EntryList entriesToKeep_) { // first get list of all images in collection StringSet images; FieldList fields = m_coll->imageFields(); EntryList allEntries = m_coll->entries(); foreach(FieldPtr field, fields) { foreach(EntryPtr entry, allEntries) { images.add(entry->field(field->name())); } foreach(EntryPtr entry, entriesToKeep_) { images.add(entry->field(field->name())); } } // now for all images not in the cache, we can clear them StringSet imagesToCheck = ImageFactory::imagesNotInCache(); // if entries_ is not empty, that means we want to limit the images removed // to those that are referenced in those entries StringSet imagesToRemove; foreach(FieldPtr field, fields) { foreach(EntryPtr entry, entries_) { QString id = entry->field(field->name()); if(!id.isEmpty() && imagesToCheck.has(id) && !images.has(id)) { imagesToRemove.add(id); } } } - const QStringList realImagesToRemove = imagesToRemove.toList(); + const QStringList realImagesToRemove = imagesToRemove.values(); for(QStringList::ConstIterator it = realImagesToRemove.begin(); it != realImagesToRemove.end(); ++it) { ImageFactory::removeImage(*it, false); // doesn't delete, just remove link } } bool Document::mergeEntry(Data::EntryPtr e1, Data::EntryPtr e2, MergeConflictResolver* resolver_) { if(!e1 || !e2) { myDebug() << "bad entry pointer"; return false; } bool ret = true; foreach(FieldPtr field, e1->collection()->fields()) { if(e2->field(field).isEmpty()) { continue; } // never try to merge entry id, creation date or mod date. Those are unique to each entry if(field->name() == QLatin1String("id") || field->name() == QLatin1String("cdate") || field->name() == QLatin1String("mdate")) { continue; } // myLog() << "reading field: " << field->name(); if(e1->field(field) == e2->field(field)) { continue; } else if(e1->field(field).isEmpty()) { // myLog() << e1->title() << ": updating field(" << field->name() << ") to " << e2->field(field); e1->setField(field, e2->field(field)); ret = true; } else if(field->type() == Data::Field::Table) { // if field F is a table-type field (album tracks, files, etc.), merge rows (keep their position) // if e1's F val in [row i, column j] empty, replace with e2's val at same position // if different (non-empty) vals at same position, CONFLICT! QStringList vals1 = FieldFormat::splitTable(e1->field(field)); QStringList vals2 = FieldFormat::splitTable(e2->field(field)); while(vals1.count() < vals2.count()) { vals1 += QString(); } for(int i = 0; i < vals2.count(); ++i) { if(vals2[i].isEmpty()) { continue; } if(vals1[i].isEmpty()) { vals1[i] = vals2[i]; ret = true; } else { QStringList parts1 = FieldFormat::splitRow(vals1[i]); QStringList parts2 = FieldFormat::splitRow(vals2[i]); bool changedPart = false; while(parts1.count() < parts2.count()) { parts1 += QString(); } for(int j = 0; j < parts2.count(); ++j) { if(parts2[j].isEmpty()) { continue; } if(parts1[j].isEmpty()) { parts1[j] = parts2[j]; changedPart = true; } else if(resolver_ && parts1[j] != parts2[j]) { int resolverResponse = resolver_->resolve(e1, e2, field, parts1[j], parts2[j]); if(resolverResponse == MergeConflictResolver::CancelMerge) { ret = false; return false; // cancel all the merge right now } else if(resolverResponse == MergeConflictResolver::KeepSecond) { parts1[j] = parts2[j]; changedPart = true; } } } if(changedPart) { vals1[i] = parts1.join(FieldFormat::columnDelimiterString()); ret = true; } } } if(ret) { e1->setField(field, vals1.join(FieldFormat::rowDelimiterString())); } // remove the merging due to user comments // maybe in the future have a more intelligent way #if 0 } else if(field->hasFlag(Data::Field::AllowMultiple)) { // if field F allows multiple values and not a Table (see above case), // e1's F values = (e1's F values) U (e2's F values) (union) // replace e1's field with union of e1's and e2's values for this field QStringList items1 = e1->fields(field, false); QStringList items2 = e2->fields(field, false); foreach(const QString& item2, items2) { // possible to have one value formatted and the other one not... if(!items1.contains(item2) && !items1.contains(Field::format(item2, field->formatType()))) { items1.append(item2); } } // not sure if I think it should be sorted or not // items1.sort(); e1->setField(field, items1.join(FieldFormat::delimiterString())); ret = true; #endif } else if(resolver_) { const int resolverResponse = resolver_->resolve(e1, e2, field); if(resolverResponse == MergeConflictResolver::CancelMerge) { ret = false; // we got cancelled return false; // cancel all the merge right now } else if(resolverResponse == MergeConflictResolver::KeepSecond) { e1->setField(field, e2->field(field)); } } else { // myDebug() << "Keeping value of" << field->name() << "for" << e1->field(QStringLiteral("title")); } } return ret; } //static QPair Document::mergeFields(Data::CollPtr coll_, Data::FieldList fields_, Data::EntryList entries_) { Data::FieldList modified, created; foreach(Data::FieldPtr field, fields_) { // don't add a field if it's a default field and not in the current collection if(coll_->hasField(field->name()) || CollectionFactory::isDefaultField(coll_->type(), field->name())) { // special case for choice fields, since we might want to add a value if(field->type() == Data::Field::Choice && coll_->hasField(field->name())) { // a2 are the existing fields in the collection, keep them in the same order QStringList a1 = coll_->fieldByName(field->name())->allowed(); foreach(const QString& newAllowedValue, field->allowed()) { if(!a1.contains(newAllowedValue)) { // could be slow for large merges, but we do only want to add new value // IF that value is actually used by an entry foreach(Data::EntryPtr entry, entries_) { if(entry->field(field->name()) == newAllowedValue) { a1 += newAllowedValue; break; } } } } if(a1.count() != coll_->fieldByName(field->name())->allowed().count()) { Data::FieldPtr f(new Data::Field(*coll_->fieldByName(field->name()))); f->setAllowed(a1); modified.append(f); } } continue; } // add field if any values are not empty foreach(Data::EntryPtr entry, entries_) { if(!entry->field(field).isEmpty()) { created.append(Data::FieldPtr(new Data::Field(*field))); break; } } } return qMakePair(modified, created); } diff --git a/src/entry.cpp b/src/entry.cpp index 7297f408..8a25adf1 100644 --- a/src/entry.cpp +++ b/src/entry.cpp @@ -1,332 +1,332 @@ /*************************************************************************** Copyright (C) 2001-2009 Robby Stephenson ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * * * ***************************************************************************/ #include "entry.h" #include "entrygroup.h" #include "collection.h" #include "field.h" #include "derivedvalue.h" #include "utils/string_utils.h" #include "utils/stringset.h" #include "tellico_debug.h" #include #include using namespace Tellico; using namespace Tellico::Data; using Tellico::Data::Entry; Entry::Entry(Tellico::Data::CollPtr coll_) : QSharedData(), m_coll(coll_), m_id(-1) { #ifndef NDEBUG if(!coll_) { myWarning() << "null collection pointer!"; } #endif } Entry::Entry(Tellico::Data::CollPtr coll_, Data::ID id_) : QSharedData(), m_coll(coll_), m_id(id_) { #ifndef NDEBUG if(!coll_) { myWarning() << "null collection pointer!"; } #endif } Entry::Entry(const Entry& entry_) : QSharedData(entry_), m_coll(entry_.m_coll), m_id(-1), m_fieldValues(entry_.m_fieldValues), m_formattedFields(entry_.m_formattedFields) { } Entry& Entry::operator=(const Entry& other_) { if(this == &other_) return *this; // static_cast(*this) = static_cast(other_); m_coll = other_.m_coll; m_id = other_.m_id; m_fieldValues = other_.m_fieldValues; m_formattedFields = other_.m_formattedFields; return *this; } Entry::~Entry() { } Tellico::Data::CollPtr Entry::collection() const { return m_coll; } void Entry::setCollection(Tellico::Data::CollPtr coll_) { if(coll_ == m_coll) { // myDebug() << "already belongs to collection!"; return; } // special case adding a book to a bibtex collection // it would be better to do this in a real OO way, but this should work const bool addEntryType = m_coll->type() == Collection::Book && coll_->type() == Collection::Bibtex && !m_coll->hasField(QStringLiteral("entry-type")); m_coll = coll_; m_id = -1; // set this after changing the m_coll pointer since setField() checks field validity if(addEntryType) { setField(QStringLiteral("entry-type"), QStringLiteral("book")); } } QString Entry::title() const { return field(QStringLiteral("title")); } QString Entry::field(const QString& fieldName_) const { return field(m_coll->fieldByName(fieldName_)); } QString Entry::field(Tellico::Data::FieldPtr field_) const { if(!field_) { return QString(); } if(field_->hasFlag(Field::Derived)) { DerivedValue dv(field_); return dv.value(EntryPtr(const_cast(this)), false); } if(m_fieldValues.contains(field_->name())) { return m_fieldValues.value(field_->name()); } return QString(); } QString Entry::formattedField(const QString& fieldName_, FieldFormat::Request request_) const { return formattedField(m_coll->fieldByName(fieldName_), request_); } QString Entry::formattedField(Tellico::Data::FieldPtr field_, FieldFormat::Request request_) const { if(!field_) { return QString(); } // don't format the value unless it's requested to do so if(request_ == FieldFormat::AsIsFormat) { return field(field_); } const FieldFormat::Type flag = field_->formatType(); if(field_->hasFlag(Field::Derived)) { DerivedValue dv(field_); // format sub fields and whole string return FieldFormat::format(dv.value(EntryPtr(const_cast(this)), true), flag, request_); } // if auto format is not set or FormatNone, then just return the value if(flag == FieldFormat::FormatNone) { return m_coll->prepareText(field(field_)); } if(!m_formattedFields.contains(field_->name())) { QString formattedValue; if(field_->type() == Field::Table) { QStringList rows; // we only format the first column foreach(const QString& row, FieldFormat::splitTable(field(field_->name()))) { QStringList columns = FieldFormat::splitRow(row); QStringList newValues; if(!columns.isEmpty()) { foreach(const QString& value, FieldFormat::splitValue(columns.at(0))) { newValues << FieldFormat::format(value, field_->formatType(), FieldFormat::DefaultFormat); } columns.replace(0, newValues.join(FieldFormat::delimiterString())); } rows << columns.join(FieldFormat::columnDelimiterString()); } formattedValue = rows.join(FieldFormat::rowDelimiterString()); } else { QStringList values; if(field_->hasFlag(Field::AllowMultiple)) { values = FieldFormat::splitValue(field(field_->name())); } else { values << field(field_->name()); } QStringList formattedValues; foreach(const QString& value, values) { formattedValues << FieldFormat::format(m_coll->prepareText(value), flag, request_); } formattedValue = formattedValues.join(FieldFormat::delimiterString()); } if(!formattedValue.isEmpty()) { m_formattedFields.insert(field_->name(), formattedValue); } return formattedValue; } // otherwise, just look it up return m_formattedFields.value(field_->name()); } bool Entry::setField(Tellico::Data::FieldPtr field_, const QString& value_, bool updateMDate_) { return setField(field_->name(), value_, updateMDate_); } // updating the modified date of the entry is expensive with the call to QDate::currentDate // when loading a collection from a file (in particular), it's faster to ignore that date bool Entry::setField(const QString& name_, const QString& value_, bool updateMDate_) { if(name_.isEmpty()) { myWarning() << "empty field name for value:" << value_; return false; } if(m_coll->fields().isEmpty()) { myDebug() << "collection has no fields, can't add -" << name_; return false; } if(!m_coll->hasField(name_)) { myDebug() << "unknown collection entry field -" << name_ << "in collection" << m_coll->title(); myDebug() << "not adding" << value_; return false; } const bool wasDifferent = field(name_) != value_;; const bool res = setFieldImpl(name_, value_); // returning true means entry was successfully modified if(res && wasDifferent && updateMDate_ && name_ != QLatin1String("mdate") && m_coll->hasField(QStringLiteral("mdate"))) { setFieldImpl(QStringLiteral("mdate"), QDate::currentDate().toString(Qt::ISODate)); } return res; } bool Entry::setFieldImpl(const QString& name_, const QString& value_) { // an empty value means remove the field if(value_.isEmpty()) { if(m_fieldValues.contains(name_)) { m_fieldValues.remove(name_); invalidateFormattedFieldValue(name_); } return true; } if(m_coll && !m_coll->isAllowed(name_, value_)) { myDebug() << "for" << name_ << ", value is not allowed -" << value_; return false; } Data::FieldPtr f = m_coll->fieldByName(name_); if(!f) { return false; } // the string store is probable only useful for fields with auto-completion or choice/number/bool bool shareType = f->type() == Field::Choice || f->type() == Field::Bool || f->type() == Field::Image || f->type() == Field::Rating || f->type() == Field::Number; if(!(f->hasFlag(Field::AllowMultiple)) && (shareType || (f->type() == Field::Line && (f->flags() & Field::AllowCompletion)))) { m_fieldValues.insert(Tellico::shareString(name_), Tellico::shareString(value_)); } else { m_fieldValues.insert(Tellico::shareString(name_), value_); } invalidateFormattedFieldValue(name_); return true; } bool Entry::addToGroup(EntryGroup* group_) { if(!group_ || m_groups.contains(group_)) { return false; } m_groups.push_back(group_); group_->append(EntryPtr(this)); return true; } bool Entry::removeFromGroup(EntryGroup* group_) { // if the removal isn't successful, just return bool success = m_groups.removeOne(group_); success = group_->removeOne(EntryPtr(this)) && success; // myDebug() << "removing from group - " << group_->fieldName() << "--" << group_->groupName(); if(!success) { myDebug() << "failed!"; } return success; } void Entry::clearGroups() { m_groups.clear(); } // this function gets called before m_groups is updated. In fact, it is used to // update that list. This is the function that actually parses the field values // and returns the list of the group names. QStringList Entry::groupNamesByFieldName(const QString& fieldName_) const { // myDebug() << fieldName_; FieldPtr f = m_coll->fieldByName(fieldName_); if(!f) { myWarning() << "no field named" << fieldName_; return QStringList(); } StringSet groups; // check table before multiple since tables are always multiple if(f->type() == Field::Table) { // we only take groups from the first column foreach(const QString& row, FieldFormat::splitTable(field(f))) { const QStringList columns = FieldFormat::splitRow(row); const QStringList values = columns.isEmpty() ? QStringList() : FieldFormat::splitValue(columns.at(0)); foreach(const QString& value, values) { groups.add(FieldFormat::format(value, f->formatType(), FieldFormat::DefaultFormat)); } } } else if(f->hasFlag(Field::AllowMultiple)) { // use a string split instead of regexp split, since we've already enforced the space after the semi-comma groups.add(FieldFormat::splitValue(formattedField(f), FieldFormat::StringSplit)); } else { groups.add(formattedField(f)); } // possible to be empty for no value // but we want to populate an empty group - return groups.isEmpty() ? QStringList(QString()) : groups.toList(); + return groups.isEmpty() ? QStringList(QString()) : groups.values(); } bool Entry::isOwned() { return (m_coll && m_id > -1 && m_coll->entryCount() > 0 && m_coll->entries().contains(EntryPtr(this))); } // an empty string means invalidate all void Entry::invalidateFormattedFieldValue(const QString& name_) { if(name_.isEmpty()) { m_formattedFields.clear(); } else if(!m_formattedFields.isEmpty() && m_formattedFields.contains(name_)) { m_formattedFields.remove(name_); } } diff --git a/src/fetch/amazonfetcher.cpp b/src/fetch/amazonfetcher.cpp index fd580137..4037c425 100644 --- a/src/fetch/amazonfetcher.cpp +++ b/src/fetch/amazonfetcher.cpp @@ -1,1205 +1,1205 @@ /*************************************************************************** Copyright (C) 2004-2020 Robby Stephenson ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * * * ***************************************************************************/ #include #include "amazonfetcher.h" #include "amazonrequest.h" #include "../collectionfactory.h" #include "../images/imagefactory.h" #include "../utils/guiproxy.h" #include "../collection.h" #include "../entry.h" #include "../field.h" #include "../fieldformat.h" #include "../utils/string_utils.h" #include "../utils/isbnvalidator.h" #include "../gui/combobox.h" #include "../tellico_debug.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace { static const int AMAZON_RETURNS_PER_REQUEST = 10; static const int AMAZON_MAX_RETURNS_TOTAL = 20; static const char* AMAZON_ASSOC_TOKEN = "tellico-20"; } using namespace Tellico; using Tellico::Fetch::AmazonFetcher; // static // see https://webservices.amazon.com/paapi5/documentation/common-request-parameters.html#host-and-region const AmazonFetcher::SiteData& AmazonFetcher::siteData(int site_) { Q_ASSERT(site_ >= 0); Q_ASSERT(site_ < XX); static SiteData dataVector[16] = { { i18n("Amazon (US)"), "www.amazon.com", "us-east-1", QLatin1String("us"), i18n("United States") }, { i18n("Amazon (UK)"), "www.amazon.co.uk", "eu-west-1", QLatin1String("gb"), i18n("United Kingdom") }, { i18n("Amazon (Germany)"), "www.amazon.de", "eu-west-1", QLatin1String("de"), i18n("Germany") }, { i18n("Amazon (Japan)"), "www.amazon.co.jp", "us-west-2", QLatin1String("jp"), i18n("Japan") }, { i18n("Amazon (France)"), "www.amazon.fr", "eu-west-1", QLatin1String("fr"), i18n("France") }, { i18n("Amazon (Canada)"), "www.amazon.ca", "us-east-1", QLatin1String("ca"), i18n("Canada") }, { // TODO: no chinese in PAAPI-5 yet? i18n("Amazon (China)"), "www.amazon.cn", "us-west-2", QLatin1String("ch"), i18n("China") }, { i18n("Amazon (Spain)"), "www.amazon.es", "eu-west-1", QLatin1String("es"), i18n("Spain") }, { i18n("Amazon (Italy)"), "www.amazon.it", "eu-west-1", QLatin1String("it"), i18n("Italy") }, { i18n("Amazon (Brazil)"), "www.amazon.com.br", "us-east-1", QLatin1String("br"), i18n("Brazil") }, { i18n("Amazon (Australia)"), "www.amazon.com.au", "us-west-2", QLatin1String("au"), i18n("Australia") }, { i18n("Amazon (India)"), "www.amazon.in", "eu-west-1", QLatin1String("in"), i18n("India") }, { i18n("Amazon (Mexico)"), "www.amazon.com.mx", "us-east-1", QLatin1String("mx"), i18n("Mexico") }, { i18n("Amazon (Turkey)"), "www.amazon.com.tr", "eu-west-1", QLatin1String("tr"), i18n("Turkey") }, { i18n("Amazon (Singapore)"), "www.amazon.sg", "us-west-2", QLatin1String("sg"), i18n("Singapore") }, { i18n("Amazon (UAE)"), "www.amazon.ae", "eu-west-1", QLatin1String("ae"), i18n("United Arab Emirates") } }; return dataVector[qBound(0, site_, static_cast(sizeof(dataVector)/sizeof(SiteData)))]; } AmazonFetcher::AmazonFetcher(QObject* parent_) : Fetcher(parent_), m_site(Unknown), m_imageSize(MediumImage), m_assoc(QLatin1String(AMAZON_ASSOC_TOKEN)), m_limit(AMAZON_MAX_RETURNS_TOTAL), m_countOffset(0), m_page(1), m_total(-1), m_numResults(0), m_job(nullptr), m_started(false) { // to facilitate transition to Amazon PAAPI5, allow users to enable logging the Amazon // results so they can be shared for debugging const QByteArray enableLog = qgetenv("TELLICO_ENABLE_AMAZON_LOG").trimmed().toLower(); m_enableLog = (enableLog == "true" || enableLog == "1"); } AmazonFetcher::~AmazonFetcher() { } QString AmazonFetcher::source() const { return m_name.isEmpty() ? defaultName() : m_name; } QString AmazonFetcher::attribution() const { return i18n("This data is licensed under specific terms.", QLatin1String("https://affiliate-program.amazon.com/gp/advertising/api/detail/agreement.html")); } bool AmazonFetcher::canFetch(int type) const { return type == Data::Collection::Book || type == Data::Collection::ComicBook || type == Data::Collection::Bibtex || type == Data::Collection::Album || type == Data::Collection::Video || type == Data::Collection::Game || type == Data::Collection::BoardGame; } bool AmazonFetcher::canSearch(FetchKey k) const { // no UPC in Canada return k == Title || k == Person || k == ISBN || k == UPC || k == Keyword; } void AmazonFetcher::readConfigHook(const KConfigGroup& config_) { const int site = config_.readEntry("Site", int(Unknown)); Q_ASSERT(site != Unknown); m_site = static_cast(site); if(m_name.isEmpty()) { m_name = siteData(m_site).title; } QString s = config_.readEntry("AccessKey"); if(!s.isEmpty()) { m_accessKey = s; } else { myWarning() << "No Amazon access key"; } s = config_.readEntry("AssocToken"); if(!s.isEmpty()) { m_assoc = s; } s = config_.readEntry("SecretKey"); if(!s.isEmpty()) { m_secretKey = s; } else { myWarning() << "No Amazon secret key"; } int imageSize = config_.readEntry("Image Size", -1); if(imageSize > -1) { m_imageSize = static_cast(imageSize); } } void AmazonFetcher::search() { m_started = true; m_page = 1; m_total = -1; m_countOffset = 0; m_numResults = 0; doSearch(); } void AmazonFetcher::continueSearch() { m_started = true; m_limit += AMAZON_MAX_RETURNS_TOTAL; doSearch(); } void AmazonFetcher::doSearch() { if(m_secretKey.isEmpty() || m_accessKey.isEmpty()) { // this message is split in two since the first half is reused later message(i18n("Access to data from Amazon.com requires an AWS Access Key ID and a Secret Key.") + QLatin1Char(' ') + i18n("Those values must be entered in the data source settings."), MessageHandler::Error); stop(); return; } const QByteArray payload = requestPayload(request()); if(payload.isEmpty()) { stop(); return; } AmazonRequest request(m_accessKey, m_secretKey); request.setHost(siteData(m_site).host); request.setRegion(siteData(m_site).region); // debugging check if(m_testResultsFile.isEmpty()) { QUrl u; u.setHost(QString::fromUtf8(siteData(m_site).host)); m_job = KIO::storedHttpPost(payload, u, KIO::HideProgressInfo); QMapIterator i(request.headers(payload)); while(i.hasNext()) { i.next(); m_job->addMetaData(QStringLiteral("customHTTPHeader"), QString::fromUtf8(i.key() + ": " + i.value())); } } else { myDebug() << "Reading" << m_testResultsFile; m_job = KIO::storedGet(QUrl::fromLocalFile(m_testResultsFile), KIO::NoReload, KIO::HideProgressInfo); } KJobWidgets::setWindow(m_job, GUI::Proxy::widget()); connect(m_job.data(), &KJob::result, this, &AmazonFetcher::slotComplete); } void AmazonFetcher::stop() { if(!m_started) { return; } // myDebug(); if(m_job) { m_job->kill(); m_job = nullptr; } m_started = false; emit signalDone(this); } void AmazonFetcher::slotComplete(KJob*) { if(m_job->error()) { m_job->uiDelegate()->showErrorMessage(); stop(); return; } const QByteArray data = m_job->data(); if(data.isEmpty()) { myDebug() << "no data"; stop(); return; } // since the fetch is done, don't worry about holding the job pointer m_job = nullptr; if(m_enableLog) { QTemporaryFile logFile(QDir::tempPath() + QStringLiteral("/amazon-search-items-XXXXXX.json")); logFile.setAutoRemove(false); if(logFile.open()) { QTextStream t(&logFile); t.setCodec("UTF-8"); t << data; myLog() << "Writing Amazon data output to" << logFile.fileName(); } } #if 0 myWarning() << "Remove debug from amazonfetcher.cpp"; QFile f(QString::fromLatin1("/tmp/test%1.json").arg(m_page)); if(f.open(QIODevice::WriteOnly)) { QTextStream t(&f); t.setCodec("UTF-8"); t << data; } f.close(); #endif QJsonParseError jsonError; QJsonObject databject = QJsonDocument::fromJson(data, &jsonError).object(); if(jsonError.error != QJsonParseError::NoError) { myDebug() << "AmazonFetcher: JSON error -" << jsonError.errorString(); message(jsonError.errorString(), MessageHandler::Error); stop(); return; } QJsonObject resultObject = databject.value(QStringLiteral("SearchResult")).toObject(); if(resultObject.isEmpty()) { resultObject = databject.value(QStringLiteral("ItemsResult")).toObject(); } if(m_total == -1) { int totalResults = resultObject.value(QStringLiteral("TotalResultCount")).toInt(); if(totalResults > 0) { m_total = totalResults; // myDebug() << "Total results is" << totalResults; } } QStringList errors; QJsonValue errorValue = databject.value(QLatin1String("Errors")); if(!errorValue.isNull()) { foreach(const QJsonValue& error, errorValue.toArray()) { errors += error.toObject().value(QLatin1String("Message")).toString(); } } if(!errors.isEmpty()) { for(QStringList::ConstIterator it = errors.constBegin(); it != errors.constEnd(); ++it) { myDebug() << "AmazonFetcher::" << *it; } message(errors[0], MessageHandler::Error); stop(); return; } Data::CollPtr coll = createCollection(); if(!coll) { myDebug() << "no collection pointer"; stop(); return; } int count = -1; foreach(const QJsonValue& item, resultObject.value(QLatin1String("Items")).toArray()) { ++count; if(m_numResults >= m_limit) { break; } if(!m_started) { // might get aborted break; } Data::EntryPtr entry(new Data::Entry(coll)); populateEntry(entry, item.toObject()); // special case book author // amazon is really bad about not putting spaces after periods if(coll->type() == Data::Collection::Book) { QRegExp rx(QLatin1String("\\.([^\\s])")); QStringList values = FieldFormat::splitValue(entry->field(QStringLiteral("author"))); for(QStringList::Iterator it = values.begin(); it != values.end(); ++it) { (*it).replace(rx, QStringLiteral(". \\1")); } entry->setField(QStringLiteral("author"), values.join(FieldFormat::delimiterString())); } // UK puts the year in the title for some reason if(m_site == UK && coll->type() == Data::Collection::Video) { QRegExp rx(QLatin1String("\\[(\\d{4})\\]")); QString t = entry->title(); if(rx.indexIn(t) > -1) { QString y = rx.cap(1); t = t.remove(rx).simplified(); entry->setField(QStringLiteral("title"), t); if(entry->field(QStringLiteral("year")).isEmpty()) { entry->setField(QStringLiteral("year"), y); } } } // myDebug() << entry->title(); FetchResult* r = new FetchResult(Fetcher::Ptr(this), entry); m_entries.insert(r->uid, entry); emit signalResultFound(r); ++m_numResults; } // we might have gotten aborted if(!m_started) { return; } // are there any additional results to get? m_hasMoreResults = m_testResultsFile.isEmpty() && (m_page * AMAZON_RETURNS_PER_REQUEST < m_total); const int currentTotal = qMin(m_total, m_limit); if(m_testResultsFile.isEmpty() && (m_page * AMAZON_RETURNS_PER_REQUEST < currentTotal)) { int foundCount = (m_page-1) * AMAZON_RETURNS_PER_REQUEST + coll->entryCount(); message(i18n("Results from %1: %2/%3", source(), foundCount, m_total), MessageHandler::Status); ++m_page; m_countOffset = 0; doSearch(); } else if(request().value.count(QLatin1Char(';')) > 9) { // start new request after cutting off first 10 isbn values FetchRequest newRequest = request(); newRequest.value = request().value.section(QLatin1Char(';'), 10); startSearch(newRequest); } else { m_countOffset = m_entries.count() % AMAZON_RETURNS_PER_REQUEST; if(m_countOffset == 0) { ++m_page; // need to go to next page } stop(); } } Tellico::Data::EntryPtr AmazonFetcher::fetchEntryHook(uint uid_) { Data::EntryPtr entry = m_entries[uid_]; if(!entry) { myWarning() << "no entry in dict"; return entry; } // do what we can to remove useless keywords const int type = collectionType(); switch(type) { case Data::Collection::Book: case Data::Collection::ComicBook: case Data::Collection::Bibtex: if(optionalFields().contains(QStringLiteral("keyword"))) { StringSet newWords; const QStringList keywords = FieldFormat::splitValue(entry->field(QStringLiteral("keyword"))); foreach(const QString& keyword, keywords) { if(keyword == QLatin1String("General") || keyword == QLatin1String("Subjects") || keyword == QLatin1String("Par prix") || // french stuff keyword == QLatin1String("Divers") || // french stuff keyword.startsWith(QLatin1Char('(')) || keyword.startsWith(QLatin1String("Authors"))) { continue; } newWords.add(keyword); } - entry->setField(QStringLiteral("keyword"), newWords.toList().join(FieldFormat::delimiterString())); + entry->setField(QStringLiteral("keyword"), newWords.values().join(FieldFormat::delimiterString())); } entry->setField(QStringLiteral("comments"), Tellico::decodeHTML(entry->field(QStringLiteral("comments")))); break; case Data::Collection::Video: { const QString genres = QStringLiteral("genre"); QStringList oldWords = FieldFormat::splitValue(entry->field(genres)); StringSet words; // only care about genres that have "Genres" in the amazon response // and take the first word after that for(QStringList::Iterator it = oldWords.begin(); it != oldWords.end(); ++it) { if((*it).indexOf(QLatin1String("Genres")) == -1) { continue; } // the amazon2tellico stylesheet separates words with '/' QStringList nodes = (*it).split(QLatin1Char('/')); for(QStringList::Iterator it2 = nodes.begin(); it2 != nodes.end(); ++it2) { if(*it2 != QLatin1String("Genres")) { continue; } ++it2; if(it2 != nodes.end() && *it2 != QLatin1String("General")) { words.add(*it2); } break; // we're done } } - entry->setField(genres, words.toList().join(FieldFormat::delimiterString())); + entry->setField(genres, words.values().join(FieldFormat::delimiterString())); // language tracks get duplicated, too words.clear(); words.add(FieldFormat::splitValue(entry->field(QStringLiteral("language")))); - entry->setField(QStringLiteral("language"), words.toList().join(FieldFormat::delimiterString())); + entry->setField(QStringLiteral("language"), words.values().join(FieldFormat::delimiterString())); } entry->setField(QStringLiteral("plot"), Tellico::decodeHTML(entry->field(QStringLiteral("plot")))); break; case Data::Collection::Album: { const QString genres = QStringLiteral("genre"); QStringList oldWords = FieldFormat::splitValue(entry->field(genres)); StringSet words; // only care about genres that have "Styles" in the amazon response // and take the first word after that for(QStringList::Iterator it = oldWords.begin(); it != oldWords.end(); ++it) { if((*it).indexOf(QLatin1String("Styles")) == -1) { continue; } // the amazon2tellico stylesheet separates words with '/' QStringList nodes = (*it).split(QLatin1Char('/')); bool isStyle = false; for(QStringList::Iterator it2 = nodes.begin(); it2 != nodes.end(); ++it2) { if(!isStyle) { if(*it2 == QLatin1String("Styles")) { isStyle = true; } continue; } if(*it2 != QLatin1String("General")) { words.add(*it2); } } } - entry->setField(genres, words.toList().join(FieldFormat::delimiterString())); + entry->setField(genres, words.values().join(FieldFormat::delimiterString())); } entry->setField(QStringLiteral("comments"), Tellico::decodeHTML(entry->field(QStringLiteral("comments")))); break; case Data::Collection::Game: entry->setField(QStringLiteral("description"), Tellico::decodeHTML(entry->field(QStringLiteral("description")))); break; } // clean up the title parseTitle(entry); // also sometimes table fields have rows but no values Data::FieldList fields = entry->collection()->fields(); QRegExp blank(QLatin1String("[\\s") + FieldFormat::columnDelimiterString() + FieldFormat::delimiterString() + QLatin1String("]+")); // only white space, column separators and value separators foreach(Data::FieldPtr fIt, fields) { if(fIt->type() != Data::Field::Table) { continue; } if(blank.exactMatch(entry->field(fIt))) { entry->setField(fIt, QString()); } } // don't want to show image urls in the fetch dialog // so clear them after reading the URL QString imageURL; switch(m_imageSize) { case SmallImage: imageURL = entry->field(QStringLiteral("small-image")); entry->setField(QStringLiteral("small-image"), QString()); break; case MediumImage: imageURL = entry->field(QStringLiteral("medium-image")); entry->setField(QStringLiteral("medium-image"), QString()); break; case LargeImage: imageURL = entry->field(QStringLiteral("large-image")); entry->setField(QStringLiteral("large-image"), QString()); break; case NoImage: default: break; } if(!imageURL.isEmpty()) { // myDebug() << "grabbing " << imageURL; QString id = ImageFactory::addImage(QUrl::fromUserInput(imageURL), true); if(id.isEmpty()) { message(i18n("The cover image could not be loaded."), MessageHandler::Warning); } else { // amazon serves up 1x1 gifs occasionally, but that's caught in the image constructor // all relevant collection types have cover fields entry->setField(QStringLiteral("cover"), id); } } return entry; } Tellico::Fetch::FetchRequest AmazonFetcher::updateRequest(Data::EntryPtr entry_) { const int type = entry_->collection()->type(); const QString t = entry_->field(QStringLiteral("title")); if(type == Data::Collection::Book || type == Data::Collection::ComicBook || type == Data::Collection::Bibtex) { const QString isbn = entry_->field(QStringLiteral("isbn")); if(!isbn.isEmpty()) { return FetchRequest(Fetch::ISBN, isbn); } const QString a = entry_->field(QStringLiteral("author")); if(!a.isEmpty()) { return t.isEmpty() ? FetchRequest(Fetch::Person, a) : FetchRequest(Fetch::Keyword, t + QLatin1Char('-') + a); } } else if(type == Data::Collection::Album) { const QString a = entry_->field(QStringLiteral("artist")); if(!a.isEmpty()) { return t.isEmpty() ? FetchRequest(Fetch::Person, a) : FetchRequest(Fetch::Keyword, t + QLatin1Char('-') + a); } } // optimistically try searching for title and rely on Collection::sameEntry() to figure things out if(!t.isEmpty()) { return FetchRequest(Fetch::Title, t); } return FetchRequest(); } QByteArray AmazonFetcher::requestPayload(FetchRequest request_) { QJsonObject payload; payload.insert(QLatin1String("PartnerTag"), m_assoc); payload.insert(QLatin1String("PartnerType"), QLatin1String("Associates")); payload.insert(QLatin1String("Operation"), QLatin1String("SearchItems")); payload.insert(QLatin1String("SortBy"), QLatin1String("Relevance")); // not mandatory // payload.insert(QLatin1String("Marketplace"), QLatin1String(siteData(m_site).host)); if(m_page > 1) { payload.insert(QLatin1String("ItemPage"), m_page); } QJsonArray resources; resources.append(QLatin1String("ItemInfo.Title")); resources.append(QLatin1String("ItemInfo.ContentInfo")); resources.append(QLatin1String("ItemInfo.ByLineInfo")); resources.append(QLatin1String("ItemInfo.TechnicalInfo")); const int type = request_.collectionType; switch(type) { case Data::Collection::Book: case Data::Collection::ComicBook: case Data::Collection::Bibtex: payload.insert(QLatin1String("SearchIndex"), QLatin1String("Books")); resources.append(QLatin1String("ItemInfo.ExternalIds")); resources.append(QLatin1String("ItemInfo.ManufactureInfo")); break; case Data::Collection::Album: payload.insert(QLatin1String("SearchIndex"), QLatin1String("Music")); break; case Data::Collection::Video: // CA and JP appear to have a bug where Video only returns VHS or Music results // DVD will return DVD, Blu-ray, etc. so just ignore VHS for those users payload.insert(QLatin1String("SearchIndex"), QLatin1String("MoviesAndTV")); if(m_site == CA || m_site == JP || m_site == IT || m_site == ES) { payload.insert(QStringLiteral("SearchIndex"), QStringLiteral("DVD")); } else { payload.insert(QStringLiteral("SearchIndex"), QStringLiteral("Video")); } // params.insert(QStringLiteral("SortIndex"), QStringLiteral("relevancerank")); resources.append(QLatin1String("ItemInfo.ContentRating")); break; case Data::Collection::Game: payload.insert(QLatin1String("SearchIndex"), QLatin1String("VideoGames")); break; case Data::Collection::BoardGame: payload.insert(QLatin1String("SearchIndex"), QLatin1String("ToysAndGames")); // params.insert(QStringLiteral("SortIndex"), QStringLiteral("relevancerank")); break; case Data::Collection::Coin: case Data::Collection::Stamp: case Data::Collection::Wine: case Data::Collection::Base: case Data::Collection::Card: myDebug() << "can't fetch this type:" << collectionType(); return QByteArray(); } switch(request_.key) { case Title: payload.insert(QLatin1String("Title"), request_.value); break; case Person: if(type == Data::Collection::Video) { payload.insert(QStringLiteral("Actor"), request_.value); // payload.insert(QStringLiteral("Director"), request_.value); } else if(type == Data::Collection::Album) { payload.insert(QStringLiteral("Artist"), request_.value); } else if(type == Data::Collection::Book) { payload.insert(QLatin1String("Author"), request_.value); } else { payload.insert(QLatin1String("Keywords"), request_.value); } break; case ISBN: { QString cleanValue = request_.value; cleanValue.remove(QLatin1Char('-')); // ISBN only get digits or 'X' QStringList isbns = FieldFormat::splitValue(cleanValue); // Amazon isbn13 search is still very flaky, so if possible, we're going to convert // all of them to isbn10. If we run into a 979 isbn13, then we're forced to do an // isbn13 search bool isbn13 = false; for(QStringList::Iterator it = isbns.begin(); it != isbns.end(); ) { if((*it).startsWith(QLatin1String("979"))) { isbn13 = true; break; } ++it; } // if we want isbn10, then convert all if(!isbn13) { for(QStringList::Iterator it = isbns.begin(); it != isbns.end(); ++it) { if((*it).length() > 12) { (*it) = ISBNValidator::isbn10(*it); (*it).remove(QLatin1Char('-')); } } } // limit to first 10 while(isbns.size() > 10) { isbns.pop_back(); } payload.insert(QLatin1String("Keywords"), isbns.join(QLatin1String("|"))); if(isbn13) { // params.insert(QStringLiteral("IdType"), QStringLiteral("EAN")); } } break; case UPC: { QString cleanValue = request_.value; cleanValue.remove(QLatin1Char('-')); // for EAN values, add 0 to beginning if not 13 characters // in order to assume US country code from UPC value QStringList values; foreach(const QString& splitValue, cleanValue.split(FieldFormat::delimiterString())) { QString tmpValue = splitValue; if(m_site != US && tmpValue.length() == 12) { tmpValue.prepend(QLatin1Char('0')); } values << tmpValue; // limit to first 10 values if(values.length() >= 10) { break; } } payload.insert(QLatin1String("Keywords"), values.join(QLatin1String("|"))); } break; case Keyword: payload.insert(QLatin1String("Keywords"), request_.value); break; case Raw: { QString key = request_.value.section(QLatin1Char('='), 0, 0).trimmed(); QString str = request_.value.section(QLatin1Char('='), 1).trimmed(); payload.insert(key, str); } break; default: myWarning() << "key not recognized: " << request().key; return QByteArray(); } switch(m_imageSize) { case SmallImage: resources.append(QLatin1String("Images.Primary.Small")); break; case MediumImage: resources.append(QLatin1String("Images.Primary.Medium")); break; case LargeImage: resources.append(QLatin1String("Images.Primary.Large")); break; case NoImage: break; } payload.insert(QLatin1String("Resources"), resources); return QJsonDocument(payload).toJson(QJsonDocument::Compact); } Tellico::Data::CollPtr AmazonFetcher::createCollection() { Data::CollPtr coll = CollectionFactory::collection(collectionType(), true); if(!coll) { return coll; } QString imageFieldName; switch(m_imageSize) { case SmallImage: imageFieldName = QStringLiteral("small-image"); break; case MediumImage: imageFieldName = QStringLiteral("medium-image"); break; case LargeImage: imageFieldName = QStringLiteral("large-image"); break; case NoImage: break; } if(!imageFieldName.isEmpty()) { Data::FieldPtr field(new Data::Field(imageFieldName, QString(), Data::Field::URL)); coll->addField(field); } if(optionalFields().contains(QStringLiteral("amazon"))) { Data::FieldPtr field(new Data::Field(QStringLiteral("amazon"), i18n("Amazon Link"), Data::Field::URL)); field->setCategory(i18n("General")); coll->addField(field); } return coll; } void AmazonFetcher::populateEntry(Data::EntryPtr entry_, const QJsonObject& info_) { QVariantMap itemMap = info_.value(QLatin1String("ItemInfo")).toObject().toVariantMap(); entry_->setField(QStringLiteral("title"), mapValue(itemMap, "Title", "DisplayValue")); const QString isbn = mapValue(itemMap, "ExternalIds", "ISBNs", "DisplayValues"); if(!isbn.isEmpty()) { // could be duplicate isbn10 and isbn13 values QStringList isbns = FieldFormat::splitValue(isbn, FieldFormat::StringSplit); for(QStringList::Iterator it = isbns.begin(); it != isbns.end(); ++it) { if((*it).length() > 12) { (*it) = ISBNValidator::isbn10(*it); (*it).remove(QLatin1Char('-')); } } isbns.removeDuplicates(); entry_->setField(QStringLiteral("isbn"), isbns.join(FieldFormat::delimiterString())); } QStringList actors, artists, authors, illustrators, publishers; QVariantMap byLineMap = itemMap.value(QLatin1String("ByLineInfo")).toMap(); QVariantList contribArray = byLineMap.value(QLatin1String("Contributors")).toList(); foreach(const QVariant& v, contribArray) { const QVariantMap contribMap = v.toMap(); const QString role = contribMap.value(QLatin1String("Role")).toString(); const QString name = contribMap.value(QLatin1String("Name")).toString(); if(role == QLatin1String("Actor")) { actors += name; } else if(role == QLatin1String("Artist")) { artists += name; } else if(role == QLatin1String("Author")) { authors += name; } else if(role == QLatin1String("Illustrator")) { illustrators += name; } else if(role == QLatin1String("Publisher")) { publishers += name; } } // assume for books that the manufacturer is the publishers if(collectionType() == Data::Collection::Book || collectionType() == Data::Collection::Bibtex || collectionType() == Data::Collection::ComicBook) { const QString manufacturer = byLineMap.value(QLatin1String("Manufacturer")).toMap() .value(QLatin1String("DisplayValue")).toString(); publishers += manufacturer; } actors.removeDuplicates(); artists.removeDuplicates(); authors.removeDuplicates(); illustrators.removeDuplicates(); publishers.removeDuplicates(); if(!actors.isEmpty()) { entry_->setField(QStringLiteral("cast"), actors.join(FieldFormat::delimiterString())); } if(!artists.isEmpty()) { entry_->setField(QStringLiteral("artist"), artists.join(FieldFormat::delimiterString())); } if(!authors.isEmpty()) { entry_->setField(QStringLiteral("author"), authors.join(FieldFormat::delimiterString())); } if(!illustrators.isEmpty()) { entry_->setField(QStringLiteral("illustrator"), illustrators.join(FieldFormat::delimiterString())); } if(!publishers.isEmpty()) { entry_->setField(QStringLiteral("publisher"), publishers.join(FieldFormat::delimiterString())); } QVariantMap contentMap = itemMap.value(QLatin1String("ContentInfo")).toMap(); entry_->setField(QStringLiteral("edition"), mapValue(contentMap, "Edition", "DisplayValue")); entry_->setField(QStringLiteral("pages"), mapValue(contentMap, "PagesCount", "DisplayValue")); const QString pubDate = mapValue(contentMap, "PublicationDate", "DisplayValue"); if(!pubDate.isEmpty()) { entry_->setField(QStringLiteral("pub_year"), pubDate.left(4)); } QVariantList langArray = itemMap.value(QLatin1String("ContentInfo")).toMap() .value(QStringLiteral("Languages")).toMap() .value(QStringLiteral("DisplayValues")).toList(); QStringList langs; foreach(const QVariant& v, langArray) { langs += mapValue(v.toMap(), "DisplayValue"); } langs.removeDuplicates(); langs.removeAll(QString()); entry_->setField(QStringLiteral("language"), langs.join(FieldFormat::delimiterString())); if(collectionType() == Data::Collection::Book || collectionType() == Data::Collection::Bibtex || collectionType() == Data::Collection::ComicBook) { QVariantMap classificationsMap = itemMap.value(QLatin1String("Classifications")).toMap(); QVariantMap technicalMap = itemMap.value(QLatin1String("TechnicalInfo")).toMap(); QString binding = mapValue(classificationsMap, "Binding", "DisplayValue"); if(binding.isEmpty()) { binding = mapValue(technicalMap, "Formats", "DisplayValues"); } if(binding.contains(QStringLiteral("Paperback")) && binding != QStringLiteral("Trade Paperback")) { binding = i18n("Paperback"); } else if(binding.contains(QStringLiteral("Hard"))) { // could be Hardcover or Hardback binding = i18n("Hardback"); } entry_->setField(QStringLiteral("binding"), binding); } QVariantMap imagesMap = info_.value(QLatin1String("Images")).toObject().toVariantMap(); switch(m_imageSize) { case SmallImage: entry_->setField(QStringLiteral("small-image"), mapValue(imagesMap, "Primary", "Small", "URL")); break; case MediumImage: entry_->setField(QStringLiteral("medium-image"), mapValue(imagesMap, "Primary", "Medium", "URL")); break; case LargeImage: entry_->setField(QStringLiteral("large-image"), mapValue(imagesMap, "Primary", "Large", "URL")); break; case NoImage: break; } if(optionalFields().contains(QStringLiteral("amazon"))) { entry_->setField(QStringLiteral("amazon"), mapValue(info_.toVariantMap(), "DetailPageURL")); } } void AmazonFetcher::parseTitle(Tellico::Data::EntryPtr entry_) { // assume that everything in brackets or parentheses is extra static const QRegularExpression rx(QLatin1String("[\\(\\[](.*?)[\\)\\]]")); QString title = entry_->field(QStringLiteral("title")); int pos = 0; QRegularExpressionMatch match = rx.match(title, pos); while(match.hasMatch()) { pos = match.capturedStart(); if(parseTitleToken(entry_, match.captured(1))) { title.remove(match.capturedStart(), match.capturedLength()); --pos; // search again there } match = rx.match(title, pos+1); } entry_->setField(QStringLiteral("title"), title.simplified()); } bool AmazonFetcher::parseTitleToken(Tellico::Data::EntryPtr entry_, const QString& token_) { // myDebug() << "title token:" << token_; // if res = true, then the token gets removed from the title bool res = false; if(token_.indexOf(QLatin1String("widescreen"), 0, Qt::CaseInsensitive) > -1 || token_.indexOf(i18n("Widescreen"), 0, Qt::CaseInsensitive) > -1) { entry_->setField(QStringLiteral("widescreen"), QStringLiteral("true")); // res = true; leave it in the title } else if(token_.indexOf(QLatin1String("full screen"), 0, Qt::CaseInsensitive) > -1) { // skip, but go ahead and remove from title res = true; } else if(token_.indexOf(QLatin1String("import"), 0, Qt::CaseInsensitive) > -1) { // skip, but go ahead and remove from title res = true; } if(token_.indexOf(QLatin1String("blu-ray"), 0, Qt::CaseInsensitive) > -1) { entry_->setField(QStringLiteral("medium"), i18n("Blu-ray")); res = true; } else if(token_.indexOf(QLatin1String("hd dvd"), 0, Qt::CaseInsensitive) > -1) { entry_->setField(QStringLiteral("medium"), i18n("HD DVD")); res = true; } else if(token_.indexOf(QLatin1String("vhs"), 0, Qt::CaseInsensitive) > -1) { entry_->setField(QStringLiteral("medium"), i18n("VHS")); res = true; } if(token_.indexOf(QLatin1String("director's cut"), 0, Qt::CaseInsensitive) > -1 || token_.indexOf(i18n("Director's Cut"), 0, Qt::CaseInsensitive) > -1) { entry_->setField(QStringLiteral("directors-cut"), QStringLiteral("true")); // res = true; leave it in the title } if(token_.toLower() == QLatin1String("ntsc")) { entry_->setField(QStringLiteral("format"), i18n("NTSC")); res = true; } if(token_.toLower() == QLatin1String("dvd")) { entry_->setField(QStringLiteral("medium"), i18n("DVD")); res = true; } if(token_.indexOf(QLatin1String("series"), 0, Qt::CaseInsensitive) > -1) { entry_->setField(QStringLiteral("series"), token_); res = true; } static const QRegularExpression regionRx(QLatin1String("Region [1-9]")); QRegularExpressionMatch match = regionRx.match(token_); if(match.hasMatch()) { entry_->setField(QStringLiteral("region"), i18n(match.captured().toUtf8().constData())); res = true; } if(entry_->collection()->type() == Data::Collection::Game) { Data::FieldPtr f = entry_->collection()->fieldByName(QStringLiteral("platform")); if(f && f->allowed().contains(token_)) { res = true; } } return res; } //static QString AmazonFetcher::defaultName() { return i18n("Amazon.com Web Services"); } QString AmazonFetcher::defaultIcon() { return favIcon("http://www.amazon.com"); } Tellico::StringHash AmazonFetcher::allOptionalFields() { StringHash hash; hash[QStringLiteral("keyword")] = i18n("Keywords"); hash[QStringLiteral("amazon")] = i18n("Amazon Link"); return hash; } Tellico::Fetch::ConfigWidget* AmazonFetcher::configWidget(QWidget* parent_) const { return new AmazonFetcher::ConfigWidget(parent_, this); } AmazonFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const AmazonFetcher* fetcher_/*=0*/) : Fetch::ConfigWidget(parent_) { QGridLayout* l = new QGridLayout(optionsWidget()); l->setSpacing(4); l->setColumnStretch(1, 10); int row = -1; QLabel* al = new QLabel(i18n("Registration is required for accessing the %1 data source. " "If you agree to the terms and conditions, sign " "up for an account, and enter your information below.", AmazonFetcher::defaultName(), QLatin1String("https://affiliate-program.amazon.com/gp/flex/advertising/api/sign-in.html")), optionsWidget()); al->setOpenExternalLinks(true); al->setWordWrap(true); ++row; l->addWidget(al, row, 0, 1, 2); // richtext gets weird with size al->setMinimumWidth(al->sizeHint().width()); QLabel* label = new QLabel(i18n("Access key: "), optionsWidget()); l->addWidget(label, ++row, 0); m_accessEdit = new QLineEdit(optionsWidget()); connect(m_accessEdit, &QLineEdit::textChanged, this, &ConfigWidget::slotSetModified); l->addWidget(m_accessEdit, row, 1); QString w = i18n("Access to data from Amazon.com requires an AWS Access Key ID and a Secret Key."); label->setWhatsThis(w); m_accessEdit->setWhatsThis(w); label->setBuddy(m_accessEdit); label = new QLabel(i18n("Secret key: "), optionsWidget()); l->addWidget(label, ++row, 0); m_secretKeyEdit = new QLineEdit(optionsWidget()); // m_secretKeyEdit->setEchoMode(QLineEdit::PasswordEchoOnEdit); connect(m_secretKeyEdit, &QLineEdit::textChanged, this, &ConfigWidget::slotSetModified); l->addWidget(m_secretKeyEdit, row, 1); label->setWhatsThis(w); m_secretKeyEdit->setWhatsThis(w); label->setBuddy(m_secretKeyEdit); label = new QLabel(i18n("Country: "), optionsWidget()); l->addWidget(label, ++row, 0); m_siteCombo = new GUI::ComboBox(optionsWidget()); for(int i = 0; i < XX; ++i) { const AmazonFetcher::SiteData& siteData = AmazonFetcher::siteData(i); QIcon icon(QStandardPaths::locate(QStandardPaths::GenericDataLocation, QStringLiteral("kf5/locale/countries/%1/flag.png").arg(siteData.country))); m_siteCombo->addItem(icon, siteData.countryName, i); m_siteCombo->model()->sort(0); } void (GUI::ComboBox::* activatedInt)(int) = &GUI::ComboBox::activated; connect(m_siteCombo, activatedInt, this, &ConfigWidget::slotSetModified); connect(m_siteCombo, activatedInt, this, &ConfigWidget::slotSiteChanged); l->addWidget(m_siteCombo, row, 1); w = i18n("Amazon.com provides data from several different localized sites. Choose the one " "you wish to use for this data source."); label->setWhatsThis(w); m_siteCombo->setWhatsThis(w); label->setBuddy(m_siteCombo); label = new QLabel(i18n("&Image size: "), optionsWidget()); l->addWidget(label, ++row, 0); m_imageCombo = new GUI::ComboBox(optionsWidget()); m_imageCombo->addItem(i18n("Small Image"), SmallImage); m_imageCombo->addItem(i18n("Medium Image"), MediumImage); m_imageCombo->addItem(i18n("Large Image"), LargeImage); m_imageCombo->addItem(i18n("No Image"), NoImage); connect(m_imageCombo, activatedInt, this, &ConfigWidget::slotSetModified); l->addWidget(m_imageCombo, row, 1); w = i18n("The cover image may be downloaded as well. However, too many large images in the " "collection may degrade performance."); label->setWhatsThis(w); m_imageCombo->setWhatsThis(w); label->setBuddy(m_imageCombo); label = new QLabel(i18n("&Associate's ID: "), optionsWidget()); l->addWidget(label, ++row, 0); m_assocEdit = new QLineEdit(optionsWidget()); void (QLineEdit::* textChanged)(const QString&) = &QLineEdit::textChanged; connect(m_assocEdit, textChanged, this, &ConfigWidget::slotSetModified); l->addWidget(m_assocEdit, row, 1); w = i18n("The associate's id identifies the person accessing the Amazon.com Web Services, and is included " "in any links to the Amazon.com site."); label->setWhatsThis(w); m_assocEdit->setWhatsThis(w); label->setBuddy(m_assocEdit); l->setRowStretch(++row, 10); if(fetcher_) { m_siteCombo->setCurrentData(fetcher_->m_site); m_accessEdit->setText(fetcher_->m_accessKey); m_secretKeyEdit->setText(fetcher_->m_secretKey); m_assocEdit->setText(fetcher_->m_assoc); m_imageCombo->setCurrentData(fetcher_->m_imageSize); } else { // defaults m_assocEdit->setText(QLatin1String(AMAZON_ASSOC_TOKEN)); m_imageCombo->setCurrentData(MediumImage); } addFieldsWidget(AmazonFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList()); KAcceleratorManager::manage(optionsWidget()); } void AmazonFetcher::ConfigWidget::saveConfigHook(KConfigGroup& config_) { int n = m_siteCombo->currentData().toInt(); config_.writeEntry("Site", n); QString s = m_accessEdit->text().trimmed(); if(!s.isEmpty()) { config_.writeEntry("AccessKey", s); } s = m_secretKeyEdit->text().trimmed(); if(!s.isEmpty()) { config_.writeEntry("SecretKey", s); } s = m_assocEdit->text().trimmed(); if(!s.isEmpty()) { config_.writeEntry("AssocToken", s); } n = m_imageCombo->currentData().toInt(); config_.writeEntry("Image Size", n); } QString AmazonFetcher::ConfigWidget::preferredName() const { return AmazonFetcher::siteData(m_siteCombo->currentData().toInt()).title; } void AmazonFetcher::ConfigWidget::slotSiteChanged() { emit signalName(preferredName()); } diff --git a/src/fetch/googlebookfetcher.cpp b/src/fetch/googlebookfetcher.cpp index 12eb563f..9a67d340 100644 --- a/src/fetch/googlebookfetcher.cpp +++ b/src/fetch/googlebookfetcher.cpp @@ -1,423 +1,423 @@ /*************************************************************************** Copyright (C) 2011 Robby Stephenson ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * * * ***************************************************************************/ #include "googlebookfetcher.h" #include "../collections/bookcollection.h" #include "../entry.h" #include "../images/imagefactory.h" #include "../utils/isbnvalidator.h" #include "../utils/guiproxy.h" #include "../utils/string_utils.h" #include "../core/filehandler.h" #include "../tellico_debug.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace { static const int GOOGLEBOOK_MAX_RETURNS = 20; static const char* GOOGLEBOOK_API_URL = "https://www.googleapis.com/books/v1/volumes"; static const char* GOOGLEBOOK_API_KEY = "AIzaSyBdsa_DEGpDQ6PzZyYHHHokRIBY8thOdUQ"; } using namespace Tellico; using Tellico::Fetch::GoogleBookFetcher; GoogleBookFetcher::GoogleBookFetcher(QObject* parent_) : Fetcher(parent_) , m_started(false) , m_start(0) , m_total(0) , m_apiKey(QLatin1String(GOOGLEBOOK_API_KEY)) { } GoogleBookFetcher::~GoogleBookFetcher() { } QString GoogleBookFetcher::source() const { return m_name.isEmpty() ? defaultName() : m_name; } bool GoogleBookFetcher::canSearch(FetchKey k) const { return k == Title || k == Person || k == ISBN || k == Keyword; } bool GoogleBookFetcher::canFetch(int type) const { return type == Data::Collection::Book || type == Data::Collection::Bibtex; } void GoogleBookFetcher::readConfigHook(const KConfigGroup& config_) { // allow an empty key if the config key does exist m_apiKey = config_.readEntry("API Key", GOOGLEBOOK_API_KEY); } void GoogleBookFetcher::search() { m_start = 0; m_total = -1; continueSearch(); } void GoogleBookFetcher::continueSearch() { m_started = true; // we only split ISBN and LCCN values QStringList searchTerms; if(request().key == ISBN) { searchTerms = FieldFormat::splitValue(request().value); } else { searchTerms += request().value; } foreach(const QString& searchTerm, searchTerms) { doSearch(searchTerm); } if(m_jobs.isEmpty()) { stop(); } } void GoogleBookFetcher::doSearch(const QString& term_) { QUrl u(QString::fromLatin1(GOOGLEBOOK_API_URL)); QUrlQuery q; q.addQueryItem(QStringLiteral("maxResults"), QString::number(GOOGLEBOOK_MAX_RETURNS)); q.addQueryItem(QStringLiteral("startIndex"), QString::number(m_start)); q.addQueryItem(QStringLiteral("printType"), QStringLiteral("books")); // we don't require a key, cause it might work without it if(!m_apiKey.isEmpty()) { q.addQueryItem(QStringLiteral("key"), m_apiKey); } switch(request().key) { case Title: q.addQueryItem(QStringLiteral("q"), QLatin1String("intitle:") + term_); break; case Person: q.addQueryItem(QStringLiteral("q"), QLatin1String("inauthor:") + term_); break; case ISBN: { const QString isbn = ISBNValidator::cleanValue(term_); q.addQueryItem(QStringLiteral("q"), QLatin1String("isbn:") + isbn); } break; case Keyword: q.addQueryItem(QStringLiteral("q"), term_); break; default: myWarning() << "key not recognized:" << request().key; return; } u.setQuery(q); // myDebug() << "url:" << u; QPointer job = KIO::storedGet(u, KIO::NoReload, KIO::HideProgressInfo); KJobWidgets::setWindow(job, GUI::Proxy::widget()); connect(job.data(), &KJob::result, this, &GoogleBookFetcher::slotComplete); m_jobs << job; } void GoogleBookFetcher::endJob(KIO::StoredTransferJob* job_) { m_jobs.removeOne(job_); if(m_jobs.isEmpty()) { stop(); } } void GoogleBookFetcher::stop() { if(!m_started) { return; } foreach(QPointer job, m_jobs) { if(job) { job->kill(); } } m_jobs.clear(); m_started = false; emit signalDone(this); } Tellico::Data::EntryPtr GoogleBookFetcher::fetchEntryHook(uint uid_) { Data::EntryPtr entry = m_entries.value(uid_); if(!entry) { myWarning() << "no entry in dict"; return Data::EntryPtr(); } QString gbs = entry->field(QStringLiteral("gbs-link")); if(!gbs.isEmpty()) { // quiet QByteArray data = FileHandler::readDataFile(QUrl::fromUserInput(gbs), true); QJsonDocument doc = QJsonDocument::fromJson(data); populateEntry(entry, doc.object().toVariantMap()); } const QString image_id = entry->field(QStringLiteral("cover")); // if it's still a url, we need to load it if(image_id.startsWith(QLatin1String("http"))) { const QString id = ImageFactory::addImage(QUrl::fromUserInput(image_id), true); if(id.isEmpty()) { message(i18n("The cover image could not be loaded."), MessageHandler::Warning); entry->setField(QStringLiteral("cover"), QString()); } else { entry->setField(QStringLiteral("cover"), id); } } // don't want to include gbs json link entry->setField(QStringLiteral("gbs-link"), QString()); return entry; } Tellico::Fetch::FetchRequest GoogleBookFetcher::updateRequest(Data::EntryPtr entry_) { const QString isbn = entry_->field(QStringLiteral("isbn")); if(!isbn.isEmpty()) { return FetchRequest(ISBN, isbn); } QString title = entry_->field(QStringLiteral("title")); if(!title.isEmpty()) { return FetchRequest(Title, title); } return FetchRequest(); } void GoogleBookFetcher::slotComplete(KJob* job_) { KIO::StoredTransferJob* job = static_cast(job_); // myDebug(); if(job->error()) { job->uiDelegate()->showErrorMessage(); endJob(job); return; } QByteArray data = job->data(); if(data.isEmpty()) { myDebug() << "no data"; endJob(job); return; } #if 0 myWarning() << "Remove debug from googlebookfetcher.cpp"; QFile f(QString::fromLatin1("/tmp/test.json")); if(f.open(QIODevice::WriteOnly)) { QTextStream t(&f); t.setCodec("UTF-8"); t << data; } f.close(); #endif Data::CollPtr coll(new Data::BookCollection(true)); // always add the gbs-link for fetchEntryHook Data::FieldPtr field(new Data::Field(QStringLiteral("gbs-link"), QStringLiteral("GBS Link"), Data::Field::URL)); field->setCategory(i18n("General")); coll->addField(field); if(!coll->hasField(QStringLiteral("googlebook")) && optionalFields().contains(QStringLiteral("googlebook"))) { Data::FieldPtr field(new Data::Field(QStringLiteral("googlebook"), i18n("Google Book Link"), Data::Field::URL)); field->setCategory(i18n("General")); coll->addField(field); } QJsonDocument doc = QJsonDocument::fromJson(data); QVariantMap result = doc.object().toVariantMap(); m_total = result.value(QStringLiteral("totalItems")).toInt(); // myDebug() << "total:" << m_total; QVariantList resultList = result.value(QStringLiteral("items")).toList(); if(resultList.isEmpty()) { myDebug() << "no results"; endJob(job); return; } foreach(const QVariant& result, resultList) { // myDebug() << "found result:" << result; Data::EntryPtr entry(new Data::Entry(coll)); populateEntry(entry, result.toMap()); FetchResult* r = new FetchResult(Fetcher::Ptr(this), entry); m_entries.insert(r->uid, entry); emit signalResultFound(r); } m_start = m_entries.count(); m_hasMoreResults = request().key != ISBN && m_start <= m_total; endJob(job); } void GoogleBookFetcher::populateEntry(Data::EntryPtr entry, const QVariantMap& resultMap) { if(entry->collection()->hasField(QStringLiteral("gbs-link"))) { entry->setField(QStringLiteral("gbs-link"), mapValue(resultMap, "selfLink")); } const QVariantMap volumeMap = resultMap.value(QStringLiteral("volumeInfo")).toMap(); entry->setField(QStringLiteral("title"), mapValue(volumeMap, "title")); entry->setField(QStringLiteral("subtitle"), mapValue(volumeMap, "subtitle")); entry->setField(QStringLiteral("pub_year"), mapValue(volumeMap, "publishedDate").left(4)); entry->setField(QStringLiteral("author"), mapValue(volumeMap, "authors")); // workaround for bug, where publisher can be enclosed in quotes QString pub = mapValue(volumeMap, "publisher"); if(pub.startsWith(QLatin1Char('"')) && pub.endsWith(QLatin1Char('"'))) { pub.chop(1); pub = pub.remove(0, 1); } entry->setField(QStringLiteral("publisher"), pub); entry->setField(QStringLiteral("pages"), mapValue(volumeMap, "pageCount")); entry->setField(QStringLiteral("language"), mapValue(volumeMap, "language")); entry->setField(QStringLiteral("comments"), mapValue(volumeMap, "description")); QStringList catList = volumeMap.value(QStringLiteral("categories")).toStringList(); // google is going to give us a lot of categories QSet cats; foreach(const QString& cat, catList) { cats += cat.split(QRegExp(QLatin1String("\\s*/\\s*"))).toSet(); } // remove General cats.remove(QStringLiteral("General")); - catList = cats.toList(); + catList = cats.values(); catList.sort(); entry->setField(QStringLiteral("keyword"), catList.join(FieldFormat::delimiterString())); QString isbn; foreach(const QVariant& idVariant, volumeMap.value(QLatin1String("industryIdentifiers")).toList()) { const QVariantMap idMap = idVariant.toMap(); if(mapValue(idMap, "type") == QLatin1String("ISBN_10")) { isbn = mapValue(idMap, "identifier"); break; } else if(mapValue(idMap, "type") == QLatin1String("ISBN_13")) { isbn = mapValue(idMap, "identifier"); // allow isbn10 to override, so don't break here } } if(!isbn.isEmpty()) { ISBNValidator val(this); val.fixup(isbn); entry->setField(QStringLiteral("isbn"), isbn); } const QVariantMap imageMap = volumeMap.value(QStringLiteral("imageLinks")).toMap(); if(imageMap.contains(QStringLiteral("small"))) { entry->setField(QStringLiteral("cover"), mapValue(imageMap, "small")); } else if(imageMap.contains(QStringLiteral("thumbnail"))) { entry->setField(QStringLiteral("cover"), mapValue(imageMap, "thumbnail")); } else if(imageMap.contains(QStringLiteral("smallThumbnail"))) { entry->setField(QStringLiteral("cover"), mapValue(imageMap, "smallThumbnail")); } if(optionalFields().contains(QStringLiteral("googlebook"))) { entry->setField(QStringLiteral("googlebook"), mapValue(volumeMap, "infoLink")); } } Tellico::Fetch::ConfigWidget* GoogleBookFetcher::configWidget(QWidget* parent_) const { return new GoogleBookFetcher::ConfigWidget(parent_, this); } QString GoogleBookFetcher::defaultName() { return i18n("Google Book Search"); } QString GoogleBookFetcher::defaultIcon() { return favIcon("http://books.google.com"); } Tellico::StringHash GoogleBookFetcher::allOptionalFields() { StringHash hash; hash[QStringLiteral("googlebook")] = i18n("Google Book Link"); return hash; } GoogleBookFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const GoogleBookFetcher* fetcher_) : Fetch::ConfigWidget(parent_) { QGridLayout* l = new QGridLayout(optionsWidget()); l->setSpacing(4); l->setColumnStretch(1, 10); int row = -1; QLabel* al = new QLabel(i18n("Registration is required for accessing the %1 data source. " "If you agree to the terms and conditions, sign " "up for an account, and enter your information below.", preferredName(), QLatin1String("https://code.google.com/apis/console")), optionsWidget()); al->setOpenExternalLinks(true); al->setWordWrap(true); ++row; l->addWidget(al, row, 0, 1, 2); // richtext gets weird with size al->setMinimumWidth(al->sizeHint().width()); QLabel* label = new QLabel(i18n("Access key: "), optionsWidget()); l->addWidget(label, ++row, 0); m_apiKeyEdit = new QLineEdit(optionsWidget()); connect(m_apiKeyEdit, &QLineEdit::textChanged, this, &ConfigWidget::slotSetModified); l->addWidget(m_apiKeyEdit, row, 1); QString w = i18n("The default Tellico key may be used, but searching may fail due to reaching access limits."); label->setWhatsThis(w); m_apiKeyEdit->setWhatsThis(w); label->setBuddy(m_apiKeyEdit); l->setRowStretch(++row, 10); // now add additional fields widget addFieldsWidget(GoogleBookFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList()); if(fetcher_ && fetcher_->m_apiKey != QLatin1String(GOOGLEBOOK_API_KEY)) { // only show the key if it is not the default Tellico one... // that way the user is prompted to apply for their own m_apiKeyEdit->setText(fetcher_->m_apiKey); } } void GoogleBookFetcher::ConfigWidget::saveConfigHook(KConfigGroup& config_) { QString apiKey = m_apiKeyEdit->text().trimmed(); if(!apiKey.isEmpty()) { config_.writeEntry("API Key", apiKey); } } QString GoogleBookFetcher::ConfigWidget::preferredName() const { return GoogleBookFetcher::defaultName(); } diff --git a/src/fetch/imdbfetcher.cpp b/src/fetch/imdbfetcher.cpp index ba31dfae..c0a0252b 100644 --- a/src/fetch/imdbfetcher.cpp +++ b/src/fetch/imdbfetcher.cpp @@ -1,1558 +1,1558 @@ /*************************************************************************** Copyright (C) 2004-2009 Robby Stephenson ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * * * ***************************************************************************/ #include "imdbfetcher.h" #include "../utils/guiproxy.h" #include "../collections/videocollection.h" #include "../entry.h" #include "../field.h" #include "../fieldformat.h" #include "../core/filehandler.h" #include "../images/imagefactory.h" #include "../utils/string_utils.h" #include "../tellico_debug.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace { static const uint IMDB_MAX_RESULTS = 20; } using namespace Tellico; using Tellico::Fetch::IMDBFetcher; QRegExp* IMDBFetcher::s_tagRx = nullptr; QRegExp* IMDBFetcher::s_anchorRx = nullptr; QRegExp* IMDBFetcher::s_anchorTitleRx = nullptr; QRegExp* IMDBFetcher::s_anchorNameRx = nullptr; QRegExp* IMDBFetcher::s_titleRx = nullptr; // static void IMDBFetcher::initRegExps() { s_tagRx = new QRegExp(QStringLiteral("<.*>")); s_tagRx->setMinimal(true); s_anchorRx = new QRegExp(QStringLiteral("]*href\\s*=\\s*\"([^\"]+)\"[^<]*>([^<]+)"), Qt::CaseInsensitive); s_anchorRx->setMinimal(true); s_anchorTitleRx = new QRegExp(QStringLiteral("]*href\\s*=\\s*\"([^\"]*/title/[^\"]*)\"[^<]*>([^<]*)"), Qt::CaseInsensitive); s_anchorTitleRx->setMinimal(true); s_anchorNameRx = new QRegExp(QStringLiteral("]*href\\s*=\\s*\"([^\"]*/name/[^\"]*)\"[^<]*>(.+)"), Qt::CaseInsensitive); s_anchorNameRx->setMinimal(true); s_titleRx = new QRegExp(QStringLiteral("(.*)"), Qt::CaseInsensitive); s_titleRx->setMinimal(true); } // static const IMDBFetcher::LangData& IMDBFetcher::langData(int lang_) { Q_ASSERT(lang_ >= 0); Q_ASSERT(lang_ < 6); static LangData dataVector[6] = { { i18n("Internet Movie Database"), QStringLiteral("www.imdb.com"), QStringLiteral("findSectionHeader"), QStringLiteral("Exact Matches"), QStringLiteral("Partial Matches"), QStringLiteral("Approx Matches"), QStringLiteral("findSectionHeader"), QStringLiteral("Other Results"), QStringLiteral("aka"), QStringLiteral("Directed by"), QStringLiteral("Written by"), QStringLiteral("Produced by"), QStringLiteral("runtime.*(\\d+)\\s+min"), QStringLiteral("aspect ratio"), QStringLiteral("also known as"), QStringLiteral("Production Co"), QStringLiteral("cast"), QStringLiteral("cast overview"), QStringLiteral("credited cast"), QStringLiteral("episodes"), QStringLiteral("Genre"), QStringLiteral("Sound"), QStringLiteral("Color"), QStringLiteral("Language"), QStringLiteral("Certification"), QStringLiteral("Country"), QStringLiteral("plot\\s+(outline|summary)(?!/)") }, { i18n("Internet Movie Database (French)"), QStringLiteral("www.imdb.fr"), QStringLiteral("findSectionHeader"), QStringLiteral("Résultats Exacts"), QStringLiteral("Résultats Partiels"), QStringLiteral("Résultats Approximatif"), QStringLiteral("findSectionHeader"), QStringLiteral("Résultats Autres"), QStringLiteral("autre titre"), QStringLiteral("Réalisateur"), QStringLiteral("Scénarist"), QString(), QStringLiteral("Durée.*(\\d+)\\s+min"), QStringLiteral("Format"), QStringLiteral("Alias"), QStringLiteral("Sociétés de Production"), QStringLiteral("Ensemble"), QStringLiteral("cast overview"), // couldn't get phrase QStringLiteral("credited cast"), // couldn't get phrase QStringLiteral("episodes"), QStringLiteral("Genre"), QStringLiteral("Son"), QStringLiteral("Couleur"), QStringLiteral("Langue"), QStringLiteral("Classification"), QStringLiteral("Pays"), QStringLiteral("Intrigue\\s*") }, { i18n("Internet Movie Database (Spanish)"), QStringLiteral("www.imdb.es"), QStringLiteral("findSectionHeader"), QStringLiteral("Resultados Exactos"), QStringLiteral("Resultados Parciales"), QStringLiteral("Resultados Aproximados"), QStringLiteral("findSectionHeader"), QStringLiteral("Resultados Otros"), QStringLiteral("otro título"), QStringLiteral("Director"), QStringLiteral("Escritores"), QString(), QStringLiteral("Duración.*(\\d+)\\s+min"), QStringLiteral("Relación de Aspecto"), QStringLiteral("Conocido como"), QStringLiteral("Compañías Productores"), QStringLiteral("Reparto"), QStringLiteral("cast overview"), // couldn't get phrase QStringLiteral("credited cast"), // couldn't get phrase QStringLiteral("episodes"), QStringLiteral("Género"), QStringLiteral("Sonido"), QStringLiteral("Color"), QStringLiteral("Idioma"), QStringLiteral("Clasificación"), QStringLiteral("País"), QStringLiteral("Trama\\s*") }, { i18n("Internet Movie Database (German)"), QStringLiteral("www.imdb.de"), QStringLiteral("findSectionHeader"), QStringLiteral("genaue Übereinstimmung"), QStringLiteral("teilweise Übereinstimmung"), QStringLiteral("näherungsweise Übereinstimmung"), QStringLiteral("findSectionHeader"), QStringLiteral("andere Übereinstimmung"), QStringLiteral("andere titel"), QStringLiteral("Regisseur"), QStringLiteral("Drehbuchautoren"), QString(), QStringLiteral("Länge.*(\\d+)\\s+min"), QStringLiteral("Seitenverhältnis"), QStringLiteral("Auch bekannt als"), QStringLiteral("Produktionsfirmen"), QStringLiteral("Besetzung"), QStringLiteral("cast overview"), // couldn't get phrase QStringLiteral("credited cast"), // couldn't get phrase QStringLiteral("episodes"), QStringLiteral("Genre"), QStringLiteral("Tonverfahren"), QStringLiteral("Farbe"), QStringLiteral("Sprache"), QStringLiteral("Altersfreigabe"), QStringLiteral("Land"), QStringLiteral("Handlung\\s*") }, { i18n("Internet Movie Database (Italian)"), QStringLiteral("www.imdb.it"), QStringLiteral("findSectionHeader"), QStringLiteral("risultati esatti"), QStringLiteral("risultati parziali"), QStringLiteral("risultati approssimati"), QStringLiteral("findSectionHeader"), QStringLiteral("Resultados Otros"), QStringLiteral("otro título"), QStringLiteral("Regista"), QStringLiteral("Sceneggiatori"), QString(), QStringLiteral("Durata.*(\\d+)\\s+min"), QStringLiteral("Aspect Ratio"), QStringLiteral("Alias"), QStringLiteral("Società di produzione"), QStringLiteral("Cast"), QStringLiteral("cast overview"), // couldn't get phrase QStringLiteral("credited cast"), // couldn't get phrase QStringLiteral("episodes"), QStringLiteral("Genere"), QStringLiteral("Sonoro"), QStringLiteral("Colore"), QStringLiteral("Lingua"), QStringLiteral("Divieti"), QStringLiteral("Nazionalità"), QStringLiteral("Trama\\s*") }, { i18n("Internet Movie Database (Portuguese)"), QStringLiteral("www.imdb.pt"), QStringLiteral("findSectionHeader"), QStringLiteral("Exato"), QStringLiteral("Combinação Parcial"), QStringLiteral("Combinação Aproximada"), QStringLiteral("findSectionHeader"), QStringLiteral("Combinação Otros"), QStringLiteral("otro título"), QStringLiteral("Diretor"), QStringLiteral("Escritores"), QString(), QStringLiteral("Duração.*(\\d+)\\s+min"), QStringLiteral("Resolução"), QStringLiteral("Também Conhecido Como"), QStringLiteral("Companhias de Produção"), QStringLiteral("Elenco"), QStringLiteral("cast overview"), // couldn't get phrase QStringLiteral("credited cast"), // couldn't get phrase QStringLiteral("episodes"), QStringLiteral("Gênero"), QStringLiteral("Mixagem de Som"), QStringLiteral("Cor"), QStringLiteral("Lingua"), QStringLiteral("Certificação"), QStringLiteral("País"), QStringLiteral("Argumento\\s*") } }; return dataVector[qBound(0, lang_, static_cast(sizeof(dataVector)/sizeof(LangData)))]; } IMDBFetcher::IMDBFetcher(QObject* parent_) : Fetcher(parent_), m_job(nullptr), m_started(false), m_fetchImages(true), m_numCast(10), m_redirected(false), m_limit(IMDB_MAX_RESULTS), m_lang(EN), m_currentTitleBlock(Unknown), m_countOffset(0) { if(!s_tagRx) { initRegExps(); } m_host = langData(m_lang).siteHost; } IMDBFetcher::~IMDBFetcher() { } QString IMDBFetcher::source() const { return m_name.isEmpty() ? defaultName() : m_name; } bool IMDBFetcher::canFetch(int type) const { return type == Data::Collection::Video; } // imdb can search title only bool IMDBFetcher::canSearch(FetchKey k) const { return k == Title; } void IMDBFetcher::readConfigHook(const KConfigGroup& config_) { /* const int lang = config_.readEntry("Lang", int(EN)); m_lang = static_cast(lang); */ if(m_name.isEmpty()) { m_name = langData(m_lang).siteTitle; } QString h = config_.readEntry("Host"); if(h.isEmpty()) { m_host = langData(m_lang).siteHost; } else { m_host = h; } m_numCast = config_.readEntry("Max Cast", 10); m_fetchImages = config_.readEntry("Fetch Images", true); } // multiple values not supported void IMDBFetcher::search() { m_started = true; m_redirected = false; m_matches.clear(); m_popularTitles.clear(); m_exactTitles.clear(); m_partialTitles.clear(); m_currentTitleBlock = Unknown; m_countOffset = 0; m_url = QUrl(); m_url.setScheme(QStringLiteral("https")); m_url.setHost(m_host); m_url.setPath(QStringLiteral("/find")); // as far as I can tell, the url encoding should always be iso-8859-1? QUrlQuery q; q.addQueryItem(QStringLiteral("q"), request().value); switch(request().key) { case Title: q.addQueryItem(QStringLiteral("s"), QStringLiteral("tt")); m_url.setQuery(q); break; case Raw: m_url = QUrl(request().value); break; default: myWarning() << "not supported:" << request().key; stop(); return; } // myDebug() << m_url; m_job = KIO::storedGet(m_url, KIO::NoReload, KIO::HideProgressInfo); KJobWidgets::setWindow(m_job, GUI::Proxy::widget()); connect(m_job.data(), &KJob::result, this, &IMDBFetcher::slotComplete); connect(m_job.data(), &KIO::TransferJob::redirection, this, &IMDBFetcher::slotRedirection); } void IMDBFetcher::continueSearch() { m_started = true; m_limit += IMDB_MAX_RESULTS; if(m_currentTitleBlock == Popular) { parseTitleBlock(m_popularTitles); // if the offset is 0, then we need to be looking at the next block m_currentTitleBlock = m_countOffset == 0 ? Exact : Popular; } // current title block might have changed if(m_currentTitleBlock == Exact) { parseTitleBlock(m_exactTitles); m_currentTitleBlock = m_countOffset == 0 ? Partial : Exact; } if(m_currentTitleBlock == Partial) { parseTitleBlock(m_partialTitles); m_currentTitleBlock = m_countOffset == 0 ? Approx : Partial; } if(m_currentTitleBlock == Approx) { parseTitleBlock(m_approxTitles); m_currentTitleBlock = m_countOffset == 0 ? Unknown : Approx; } stop(); } void IMDBFetcher::stop() { if(!m_started) { return; } if(m_job) { m_job->kill(); m_job = nullptr; } m_started = false; m_redirected = false; emit signalDone(this); } void IMDBFetcher::slotRedirection(KIO::Job*, const QUrl& toURL_) { m_url = toURL_; if(m_url.path().contains(QRegExp(QStringLiteral("/tt\\d+/$")))) { m_url.setPath(m_url.path() + QStringLiteral("reference")); } m_redirected = true; } void IMDBFetcher::slotComplete(KJob*) { if(m_job->error()) { m_job->uiDelegate()->showErrorMessage(); stop(); return; } m_text = Tellico::fromHtmlData(m_job->data(), "UTF-8"); if(m_text.isEmpty()) { myLog() << "No data returned"; stop(); return; } // see bug 319662. If fetcher is cancelled, job is killed // if the pointer is retained, it gets double-deleted m_job = nullptr; #if 0 myWarning() << "Remove debug from imdbfetcher.cpp for /tmp/testimdbresults.html"; QFile f(QString::fromLatin1("/tmp/testimdbresults.html")); if(f.open(QIODevice::WriteOnly)) { QTextStream t(&f); t.setCodec("UTF-8"); t << m_text; } f.close(); #endif // a single result was found if we got redirected switch(request().key) { case Title: if(m_redirected) { parseSingleTitleResult(); } else { parseMultipleTitleResults(); } break; case Raw: parseSingleTitleResult(); break; default: myWarning() << "skipping results"; break; } } void IMDBFetcher::parseSingleTitleResult() { s_titleRx->indexIn(Tellico::decodeHTML(m_text)); // split title at parenthesis const QString cap1 = s_titleRx->cap(1); int pPos = cap1.indexOf(QLatin1Char('(')); // FIXME: maybe remove parentheses here? FetchResult* r = new FetchResult(Fetcher::Ptr(this), pPos == -1 ? cap1 : cap1.left(pPos), pPos == -1 ? QString() : cap1.mid(pPos)); // IMDB returns different HTML for single title results and has a query in the url // clear the query so we download the "canonical" page for the title QUrl url(m_url); url.setQuery(QString()); m_matches.insert(r->uid, url); m_allMatches.insert(r->uid, url); emit signalResultFound(r); m_hasMoreResults = false; stop(); } void IMDBFetcher::parseMultipleTitleResults() { QString output = Tellico::decodeHTML(m_text); const LangData& data = langData(m_lang); // IMDb can return three title lists, popular, exact, and partial // the popular titles are in the first table int pos_popular = output.indexOf(data.title_popular, 0, Qt::CaseInsensitive); int pos_exact = output.indexOf(data.match_exact, qMax(pos_popular, 0), Qt::CaseInsensitive); int pos_partial = output.indexOf(data.match_partial, qMax(pos_exact, 0), Qt::CaseInsensitive); int pos_approx = output.indexOf(data.match_approx, qMax(pos_partial, 0), Qt::CaseInsensitive); int end_popular = pos_exact; // keep track of where to end if(end_popular == -1) { end_popular = pos_partial == -1 ? (pos_approx == -1 ? output.length() : pos_approx) : pos_partial; } int end_exact = pos_partial; // keep track of where to end if(end_exact == -1) { end_exact = pos_approx == -1 ? output.length() : pos_approx; } int end_partial = pos_approx; // keep track of where to end if(end_partial == -1) { end_partial = output.length(); } // if found popular matches if(pos_popular > -1) { m_popularTitles = output.mid(pos_popular, end_popular-pos_popular); } // if found exact matches if(pos_exact > -1) { m_exactTitles = output.mid(pos_exact, end_exact-pos_exact); } if(pos_partial > -1) { m_partialTitles = output.mid(pos_partial, end_partial-pos_partial); } if(pos_approx > -1) { m_approxTitles = output.mid(pos_approx); } parseTitleBlock(m_popularTitles); // if the offset is 0, then we need to be looking at the next block m_currentTitleBlock = m_countOffset == 0 ? Exact : Popular; if(m_matches.size() < m_limit) { parseTitleBlock(m_exactTitles); m_currentTitleBlock = m_countOffset == 0 ? Partial : Exact; } if(m_matches.size() < m_limit) { parseTitleBlock(m_partialTitles); m_currentTitleBlock = m_countOffset == 0 ? Approx : Partial; } if(m_matches.size() < m_limit) { parseTitleBlock(m_approxTitles); m_currentTitleBlock = m_countOffset == 0 ? Unknown : Approx; } if(m_matches.size() == 0) { myLog() << "no matches found."; } stop(); } void IMDBFetcher::parseTitleBlock(const QString& str_) { if(str_.isEmpty()) { m_countOffset = 0; return; } QRegExp akaRx(QStringLiteral("%1 (.*)(||indexIn(str_); while(m_started && start > -1) { // split title at parenthesis const QString cap1 = s_anchorTitleRx->cap(1); // the anchor url const QString cap2 = s_anchorTitleRx->cap(2).trimmed(); // the anchor text start += s_anchorTitleRx->matchedLength(); int pPos = cap2.indexOf(QLatin1Char('(')); // if it has parentheses, use that for description QString desc; if(pPos > -1) { int pPos2 = cap2.indexOf(QLatin1Char(')'), pPos+1); if(pPos2 > -1) { desc = cap2.mid(pPos+1, pPos2-pPos-1); } } else { // parenthesis might be outside anchor tag int end = s_anchorTitleRx->indexIn(str_, start); if(end == -1) { end = str_.length(); } const QString text = str_.mid(start, end-start); pPos = text.indexOf(QLatin1Char('(')); if(pPos > -1) { const int pNewLine = text.indexOf(QStringLiteral(" -1 && (pNewLine == -1 || pPos < pNewLine)) { const int pPos2 = text.indexOf(QLatin1Char(')'), pPos); desc = text.mid(pPos+1, pPos2-pPos-1); } } pPos = -1; } } // multiple matches might have 'aka' info int end = s_anchorTitleRx->indexIn(str_, start+1); if(end == -1) { end = str_.length(); } int akaPos = akaRx.indexIn(str_, start+1); if(akaPos > -1 && akaPos < end) { // limit to 50 chars desc += QLatin1Char(' ') + akaRx.cap(1).trimmed().remove(*s_tagRx); if(desc.length() > 50) { desc = desc.left(50) + QStringLiteral("..."); } } start = s_anchorTitleRx->indexIn(str_, start); if(count < m_countOffset) { ++count; continue; } // if we got this far, then there is a valid result if(m_matches.size() >= m_limit) { m_hasMoreResults = true; break; } FetchResult* r = new FetchResult(Fetcher::Ptr(this), pPos == -1 ? cap2 : cap2.left(pPos), desc); QUrl u = QUrl(m_url).resolved(QUrl(cap1)); u.setQuery(QString()); m_matches.insert(r->uid, u); m_allMatches.insert(r->uid, u); emit signalResultFound(r); ++count; } if(!m_hasMoreResults && m_currentTitleBlock != Partial) { m_hasMoreResults = true; } m_countOffset = m_matches.size() < m_limit ? 0 : count; } Tellico::Data::EntryPtr IMDBFetcher::fetchEntryHook(uint uid_) { // if we already grabbed this one, then just pull it out of the dict Data::EntryPtr entry = m_entries[uid_]; if(entry) { return entry; } if(!m_matches.contains(uid_) && !m_allMatches.contains(uid_)) { myLog() << "no url found"; return Data::EntryPtr(); } QUrl url = m_matches.contains(uid_) ? m_matches[uid_] : m_allMatches[uid_]; if(url.path().contains(QRegExp(QStringLiteral("/tt\\d+/$")))) { url.setPath(url.path() + QStringLiteral("reference")); } QUrl origURL = m_url; // keep to switch back QString results; // if the url matches the current one, no need to redownload it if(url == m_url) { // myDebug() << "matches previous URL, no downloading needed."; results = Tellico::decodeHTML(m_text); } else { // now it's synchronous // be quiet about failure results = Tellico::fromHtmlData(FileHandler::readDataFile(url, true), "UTF-8"); m_url = url; // needed for processing #if 0 myWarning() << "Remove debug from imdbfetcher.cpp for /tmp/testimdbresult.html"; myDebug() << m_url; QFile f(QStringLiteral("/tmp/testimdbresult.html")); if(f.open(QIODevice::WriteOnly)) { QTextStream t(&f); t << results; } f.close(); #endif results = Tellico::decodeHTML(results); } if(results.isEmpty()) { myLog() << "no text results"; m_url = origURL; return Data::EntryPtr(); } entry = parseEntry(results); m_url = origURL; if(!entry) { myDebug() << "error in processing entry"; return Data::EntryPtr(); } m_entries.insert(uid_, entry); // keep for later return entry; } Tellico::Data::EntryPtr IMDBFetcher::parseEntry(const QString& str_) { Data::CollPtr coll(new Data::VideoCollection(true)); Data::EntryPtr entry(new Data::Entry(coll)); doTitle(str_, entry); doRunningTime(str_, entry); doAspectRatio(str_, entry); doAlsoKnownAs(str_, entry); doPlot(str_, entry, m_url); if(m_lang == EN) { doLists(str_, entry); } else { doLists2(str_, entry); } doStudio(str_, entry); doPerson(str_, entry, langData(m_lang).director, QStringLiteral("director")); doPerson(str_, entry, langData(m_lang).writer, QStringLiteral("writer")); doRating(str_, entry); doCast(str_, entry, m_url); if(m_fetchImages) { // needs base URL doCover(str_, entry, m_url); } const QString imdb = QStringLiteral("imdb"); if(!coll->hasField(imdb) && optionalFields().contains(imdb)) { Data::FieldPtr field(new Data::Field(imdb, i18n("IMDb Link"), Data::Field::URL)); field->setCategory(i18n("General")); coll->addField(field); } if(coll->hasField(imdb) && coll->fieldByName(imdb)->type() == Data::Field::URL) { m_url.setQuery(QString()); // we want to strip the "/reference" from the url QString url = m_url.url(); if(url.endsWith(QStringLiteral("/reference"))) { url = m_url.adjusted(QUrl::RemoveFilename).url(); } entry->setField(imdb, url); } return entry; } void IMDBFetcher::doTitle(const QString& str_, Tellico::Data::EntryPtr entry_) { if(s_titleRx->indexIn(str_) > -1) { const QString cap1 = s_titleRx->cap(1); // titles always have parentheses int pPos = cap1.indexOf(QLatin1Char('(')); QString title = cap1.left(pPos).trimmed(); // remove first and last quotes is there if(title.startsWith(QLatin1Char('"')) && title.endsWith(QLatin1Char('"'))) { title = title.mid(1, title.length()-2); } entry_->setField(QStringLiteral("title"), title); // now for movies with original non-english titles, the is english // but the page header is the original title. Grab the orig title QRegExp h3TitleRx(QStringLiteral("<h3[^>]+itemprop=\"name\"\\s*>(.*)<"), Qt::CaseInsensitive); h3TitleRx.setMinimal(true); if(h3TitleRx.indexIn(str_) > -1) { const QString h3Title = h3TitleRx.cap(1).trimmed(); if(h3Title != title) { // mis-matching titles. If the user has requested original title, // put it in origtitle field and keep english as title // otherwise replace if(optionalFields().contains(QStringLiteral("origtitle"))) { Data::FieldPtr f(new Data::Field(QStringLiteral("origtitle"), i18n("Original Title"))); f->setFormatType(FieldFormat::FormatTitle); entry_->collection()->addField(f); entry_->setField(QStringLiteral("origtitle"), h3Title); } else { entry_->setField(QStringLiteral("title"), h3Title); } } } // remove parentheses and extract year int pPos2 = pPos+1; while(pPos2 < cap1.length() && cap1[pPos2].isDigit()) { ++pPos2; } QString year = cap1.mid(pPos+1, pPos2-pPos-1); if(!year.isEmpty()) { entry_->setField(QStringLiteral("year"), year); } } } void IMDBFetcher::doRunningTime(const QString& str_, Tellico::Data::EntryPtr entry_) { // running time QRegExp runtimeRx(langData(m_lang).runtime, Qt::CaseInsensitive); runtimeRx.setMinimal(true); if(runtimeRx.indexIn(str_) > -1) { entry_->setField(QStringLiteral("running-time"), runtimeRx.cap(1)); } } void IMDBFetcher::doAspectRatio(const QString& str_, Tellico::Data::EntryPtr entry_) { QRegExp rx(QStringLiteral("%1.*([\\d\\.\\,]+\\s*:\\s*[\\d\\.\\,]+)").arg(langData(m_lang).aspect_ratio), Qt::CaseInsensitive); rx.setMinimal(true); if(rx.indexIn(str_) > -1) { entry_->setField(QStringLiteral("aspect-ratio"), rx.cap(1).trimmed()); } } void IMDBFetcher::doAlsoKnownAs(const QString& str_, Tellico::Data::EntryPtr entry_) { if(!optionalFields().contains(QStringLiteral("alttitle"))) { return; } // match until next b tag // QRegExp akaRx(QStringLiteral("also known as(.*)<b(?:\\s.*)?>")); QRegExp akaRx(QStringLiteral("%1(.*)(<a|<span)[>\\s/]").arg(langData(m_lang).also_known_as), Qt::CaseInsensitive); akaRx.setMinimal(true); if(akaRx.indexIn(str_) > -1 && !akaRx.cap(1).isEmpty()) { Data::FieldPtr f = entry_->collection()->fieldByName(QStringLiteral("alttitle")); if(!f) { f = new Data::Field(QStringLiteral("alttitle"), i18n("Alternative Titles"), Data::Field::Table); f->setFormatType(FieldFormat::FormatTitle); entry_->collection()->addField(f); } // split by </li> QStringList list = akaRx.cap(1).split(QStringLiteral("</li>")); // lang could be included with [fr] // const QRegExp parRx(QStringLiteral("\\(.+\\)")); const QRegExp brackRx(QStringLiteral("\\[\\w+\\]")); const QRegExp countryRx(QStringLiteral("\\s*\\(.+\\)\\s*$")); QStringList values; for(QStringList::Iterator it = list.begin(); it != list.end(); ++it) { QString s = *it; // sometimes, the word "more" gets linked to the releaseinfo page, check that if(s.contains(QStringLiteral("releaseinfo"))) { continue; } s.remove(*s_tagRx); s.remove(brackRx); // remove country s.remove(countryRx); s.remove(QLatin1Char('"')); s = s.trimmed(); // the first value ends up being or starting with the colon after "Also known as" // I'm too lazy to figure out a better regexp if(s.startsWith(QLatin1Char(':'))) { s = s.mid(1); s = s.trimmed(); } if(!s.isEmpty()) { values += s; } } if(!values.isEmpty()) { entry_->setField(QStringLiteral("alttitle"), values.join(FieldFormat::rowDelimiterString())); } // } else { // myLog() << "'Also Known As' not found"; } } void IMDBFetcher::doPlot(const QString& str_, Tellico::Data::EntryPtr entry_, const QUrl& baseURL_) { // plot summaries provided by users are on a separate page // should those be preferred? bool useUserSummary = false; // match until next <p> tag QString plotRxStr = langData(m_lang).plot + QStringLiteral("(.*)</(p|div|li)"); QRegExp plotRx(plotRxStr, Qt::CaseInsensitive); plotRx.setMinimal(true); QRegExp plotURLRx(QStringLiteral("<a\\s+.*href\\s*=\\s*\".*/title/.*/plotsummary\""), Qt::CaseInsensitive); plotURLRx.setMinimal(true); if(plotRx.indexIn(str_) > -1) { QString thisPlot = plotRx.cap(2); // if ends with "Written by", remove it. It has an em tag thisPlot.remove(QRegExp(QStringLiteral("<em class=\"nobr\".*</em>"))); thisPlot.remove(*s_tagRx); // remove HTML tags thisPlot = thisPlot.simplified(); // if thisPlot ends with (more) or contains // a url that ends with plotsummary, then we'll grab it, otherwise not if(plotRx.cap(0).endsWith(QStringLiteral("(more)</")) || plotURLRx.indexIn(plotRx.cap(0)) > -1 || thisPlot.isEmpty()) { useUserSummary = true; } else { entry_->setField(QStringLiteral("plot"), thisPlot); } } else { useUserSummary = true; } if(useUserSummary) { QRegExp idRx(QStringLiteral("title/(tt\\d+)")); idRx.indexIn(baseURL_.path()); QUrl plotURL = baseURL_; plotURL.setPath(QStringLiteral("/title/") + idRx.cap(1) + QStringLiteral("/plotsummary")); // be quiet about failure QString plotPage = Tellico::fromHtmlData(FileHandler::readDataFile(plotURL, true), "UTF-8"); if(!plotPage.isEmpty()) { QRegExp plotRx(QStringLiteral("id=\"plot-summaries-content\">(.*)</p")); plotRx.setMinimal(true); QRegExp plotRx2(QStringLiteral("<div\\s+id\\s*=\\s*\"swiki.2.1\">(.*)</d")); plotRx2.setMinimal(true); QString userPlot; if(plotRx.indexIn(plotPage) > -1) { userPlot = plotRx.cap(1); } else if(plotRx2.indexIn(plotPage) > -1) { userPlot = plotRx2.cap(1); } userPlot.remove(*s_tagRx); // remove HTML tags // remove last little "written by", if there userPlot.remove(QRegExp(QStringLiteral("\\s*written by.*$"), Qt::CaseInsensitive)); if(!userPlot.isEmpty()) { entry_->setField(QStringLiteral("plot"), Tellico::decodeHTML(userPlot.simplified())); } } } // myDebug() << "Plot:" << entry_->field(QStringLiteral("plot")); } void IMDBFetcher::doStudio(const QString& str_, Tellico::Data::EntryPtr entry_) { // match until next opening tag // QRegExp productionRx(langData(m_lang).studio, Qt::CaseInsensitive); QRegExp productionRx(langData(m_lang).studio); productionRx.setMinimal(true); QRegExp blackcatRx(QStringLiteral("blackcatheader"), Qt::CaseInsensitive); blackcatRx.setMinimal(true); const int pos1 = str_.indexOf(productionRx); if(pos1 == -1) { // myLog() << "No studio found"; return; } int pos2 = str_.indexOf(blackcatRx, pos1); if(pos2 == -1) { pos2 = str_.length(); } // stop matching when getting to Distributors int pos3 = str_.indexOf(QStringLiteral("Distributors"), pos1); if(pos3 > -1 && pos3 < pos2) { pos2 = pos3; } const QString text = str_.mid(pos1, pos2-pos1); const QString company = QStringLiteral("/company/"); QStringList studios; for(int pos = s_anchorRx->indexIn(text); pos > -1; pos = s_anchorRx->indexIn(text, pos+s_anchorRx->matchedLength())) { const QString cap1 = s_anchorRx->cap(1); if(cap1.contains(company)) { studios += s_anchorRx->cap(2).trimmed(); } } entry_->setField(QStringLiteral("studio"), studios.join(FieldFormat::delimiterString())); } void IMDBFetcher::doPerson(const QString& str_, Tellico::Data::EntryPtr entry_, const QString& imdbHeader_, const QString& fieldName_) { QRegExp br2Rx(QStringLiteral("<br[\\s/]*>\\s*<br[\\s/]*>"), Qt::CaseInsensitive); br2Rx.setMinimal(true); QRegExp divRx(QStringLiteral("<div\\s[^>]*class\\s*=\\s*\"(?:ipl-header__content|info|txt-block)\"[^>]*>(.*)</table"), Qt::CaseInsensitive); divRx.setMinimal(true); const QString name = QStringLiteral("/name/"); StringSet people; for(int pos = str_.indexOf(divRx); pos > -1; pos = str_.indexOf(divRx, pos+divRx.matchedLength())) { const QString infoBlock = divRx.cap(1); if(infoBlock.contains(imdbHeader_, Qt::CaseInsensitive)) { int pos2 = s_anchorRx->indexIn(infoBlock); while(pos2 > -1) { if(s_anchorRx->cap(1).contains(name)) { people.add(s_anchorRx->cap(2).trimmed()); } pos2 = s_anchorRx->indexIn(infoBlock, pos2+s_anchorRx->matchedLength()); } break; } } if(!people.isEmpty()) { - entry_->setField(fieldName_, people.toList().join(FieldFormat::delimiterString())); + entry_->setField(fieldName_, people.values().join(FieldFormat::delimiterString())); } } void IMDBFetcher::doCast(const QString& str_, Tellico::Data::EntryPtr entry_, const QUrl& baseURL_) { // the extended cast list is on a separate page // that's usually a lot of people // but since it can be in billing order, the main actors might not // be in the short list QRegExp idRx(QStringLiteral("title/(tt\\d+)")); idRx.indexIn(baseURL_.path()); QUrl castURL = baseURL_; castURL.setPath(QStringLiteral("/title/") + idRx.cap(1) + QStringLiteral("/fullcredits")); // be quiet about failure and be sure to translate entities const QString castPage = Tellico::decodeHTML(FileHandler::readTextFile(castURL, true)); #if 0 myWarning() << "Remove debug from imdbfetcher.cpp (/tmp/testimdbcast.html)"; QFile f(QString::fromLatin1("/tmp/testimdbcast.html")); if(f.open(QIODevice::WriteOnly)) { QTextStream t(&f); t << castPage; } f.close(); #endif const LangData& data = langData(m_lang); int pos = -1; // the text to search, depends on which page is being read QString castText = castPage; if(castText.isEmpty()) { // fall back to short list castText = str_; pos = castText.indexOf(data.cast1, 0, Qt::CaseInsensitive); if(pos == -1) { pos = castText.indexOf(data.cast2, 0, Qt::CaseInsensitive); } } else { // first look for anchor QRegExp castAnchorRx(QStringLiteral("<a\\s+name\\s*=\\s*\"cast\""), Qt::CaseInsensitive); pos = castAnchorRx.indexIn(castText); if(pos < 0) { QRegExp tableClassRx(QStringLiteral("<table\\s+class\\s*=\\s*\"cast_list\""), Qt::CaseInsensitive); pos = tableClassRx.indexIn(castText); if(pos < 0) { // fragile, the word "cast" appears in the title, but need to find // the one right above the actual cast table // for TV shows, there's a link on the sidebar for "episodes case" // so need to not match that one const QString castEnd = data.cast + QStringLiteral("</"); pos = castText.indexOf(castEnd, 0, Qt::CaseInsensitive); if(pos > 9) { // back up 9 places if(castText.midRef(pos-9, 9).startsWith(data.episodes)) { // find next cast list pos = castText.indexOf(castEnd, pos+6, Qt::CaseInsensitive); } } } } } if(pos == -1) { // no cast list found myLog() << "no cast list found"; return; } // loop until closing table tag int endPos = castText.indexOf(QStringLiteral("</table"), pos, Qt::CaseInsensitive); castText = castText.mid(pos, endPos-pos+1); QStringList actorList, characterList; QRegularExpression tdActorRx(QStringLiteral("<td>.*?<a href=\"/name.+?\".*?>(.+?)</a"), QRegularExpression::DotMatchesEverythingOption); QRegularExpression tdCharRx(QStringLiteral("<td class=\"character\">(.+?)</td"), QRegularExpression::DotMatchesEverythingOption); QRegularExpressionMatchIterator i = tdActorRx.globalMatch(castText); while(i.hasNext()) { QRegularExpressionMatch match = i.next(); actorList += match.captured(1).simplified(); } i = tdCharRx.globalMatch(castText); while(i.hasNext()) { QRegularExpressionMatch match = i.next(); characterList += match.captured(1).remove(*s_tagRx).simplified(); } // sanity check while(characterList.length() > actorList.length()) { myDebug() << "Too many characters"; characterList.removeLast(); } while(characterList.length() < actorList.length()) { characterList += QString(); } QStringList cast; cast.reserve(actorList.size()); for(int i = 0; i < actorList.size(); ++i) { cast += actorList.at(i) + FieldFormat::columnDelimiterString() + characterList.at(i); if(cast.count() >= m_numCast) { break; } } if(cast.isEmpty()) { QRegExp tdRx(QStringLiteral("<td[^>]*>(.*)</td>"), Qt::CaseInsensitive); tdRx.setMinimal(true); QRegExp tdActorRx(QStringLiteral("<td\\s+[^>]*itemprop=\"actor\"[^>]*>(.*)</td>"), Qt::CaseInsensitive); tdActorRx.setMinimal(true); QRegExp tdCharRx(QStringLiteral("<td\\s+[^>]*class=\"character\"[^>]*>(.*)</td>"), Qt::CaseInsensitive); tdCharRx.setMinimal(true); pos = tdActorRx.indexIn(castText); while(pos > -1 && cast.count() < m_numCast) { QString actorText = tdActorRx.cap(1).remove(*s_tagRx).simplified(); const int pos2 = tdCharRx.indexIn(castText, pos+1); if(pos2 > -1) { cast += actorText + FieldFormat::columnDelimiterString() + tdCharRx.cap(1).remove(*s_tagRx).simplified(); } pos = tdActorRx.indexIn(castText, qMax(pos+1, pos2)); } } if(!cast.isEmpty()) { entry_->setField(QStringLiteral("cast"), cast.join(FieldFormat::rowDelimiterString())); } // also do other items from fullcredits page, like producer QStringList producers; pos = castPage.indexOf(data.producer, 0, Qt::CaseInsensitive); if(pos > -1) { int endPos = castText.indexOf(QStringLiteral("</table"), pos, Qt::CaseInsensitive); if(endPos == -1) { endPos = castText.length(); } const QString prodText = castPage.mid(pos, endPos-pos+1); QRegExp tdCharRx(QStringLiteral("<td\\s+[^>]*class=\"credit\"[^>]*>(.*)</td>")); tdCharRx.setMinimal(true); pos = s_anchorNameRx->indexIn(prodText); while(pos > -1) { const int pos2 = tdCharRx.indexIn(prodText, pos+1); const QString credit = tdCharRx.cap(1).trimmed(); if(pos2 > -1 && (credit.startsWith(QStringLiteral("producer")) || credit.startsWith(QStringLiteral("co-producer")) || credit.startsWith(QStringLiteral("associate producer")))) { producers += s_anchorNameRx->cap(2).trimmed(); } pos = s_anchorNameRx->indexIn(prodText, pos+1); } } if(!producers.isEmpty()) { entry_->setField(QStringLiteral("producer"), producers.join(FieldFormat::delimiterString())); } #if 0 myWarning() << "Remove debug from imdbfetcher.cpp"; QFile f2(QString::fromLatin1("/tmp/testimdbcast2.html")); if(f2.open(QIODevice::WriteOnly)) { QTextStream t(&f); t.setCodec("UTF-8"); t << producers.join(FieldFormat::delimiterString()); } f2.close(); #endif } void IMDBFetcher::doRating(const QString& str_, Tellico::Data::EntryPtr entry_) { if(!optionalFields().contains(QStringLiteral("imdb-rating"))) { return; } QRegExp divRx(QStringLiteral("<div class=\"ipl-rating-star[\\s\"]+>(.*)</div"), Qt::CaseInsensitive); divRx.setMinimal(true); if(divRx.indexIn(str_) > -1) { if(!entry_->collection()->hasField(QStringLiteral("imdb-rating"))) { Data::FieldPtr f(new Data::Field(QStringLiteral("imdb-rating"), i18n("IMDb Rating"), Data::Field::Rating)); f->setCategory(i18n("General")); f->setProperty(QStringLiteral("maximum"), QStringLiteral("10")); entry_->collection()->addField(f); } QString text = divRx.cap(0); text.remove(*s_tagRx); QRegExp ratingRx(QStringLiteral("\\s(\\d+.?\\d*)\\s")); if(ratingRx.indexIn(text) > -1) { bool ok; float value = ratingRx.cap(1).toFloat(&ok); if(!ok) { value = QLocale().toFloat(ratingRx.cap(1), &ok); } if(ok) { entry_->setField(QStringLiteral("imdb-rating"), QString::number(value)); } } } } void IMDBFetcher::doCover(const QString& str_, Tellico::Data::EntryPtr entry_, const QUrl& baseURL_) { QRegExp imgRx(QStringLiteral("<img\\s+[^>]*src\\s*=\\s*\"([^\"]*)\"[^>]*>"), Qt::CaseInsensitive); imgRx.setMinimal(true); QRegExp posterRx(QStringLiteral("<a\\s+[^>]*name\\s*=\\s*\"poster\"[^>]*>(.*)</a>"), Qt::CaseInsensitive); posterRx.setMinimal(true); const QString cover = QStringLiteral("cover"); int pos = posterRx.indexIn(str_); while(pos > -1) { if(posterRx.cap(1).contains(imgRx)) { QUrl u = QUrl(baseURL_).resolved(QUrl(imgRx.cap(1))); QString id = ImageFactory::addImage(u, true); if(!id.isEmpty()) { entry_->setField(cover, id); return; } } pos = posterRx.indexIn(str_, pos+posterRx.matchedLength()); } // <link rel='image_src' QRegExp linkRx(QStringLiteral("<link (.*)>"), Qt::CaseInsensitive); linkRx.setMinimal(true); const QString src = QStringLiteral("image_src"); pos = linkRx.indexIn(str_); while(pos > -1) { const QString tag = linkRx.cap(1); if(tag.contains(src, Qt::CaseInsensitive)) { QRegExp hrefRx(QStringLiteral("href=['\"](.*)['\"]"), Qt::CaseInsensitive); hrefRx.setMinimal(true); if(hrefRx.indexIn(tag) > -1) { QUrl u = QUrl(baseURL_).resolved(QUrl(hrefRx.cap(1))); // imdb uses amazon media image, where the img src "encodes" requests for image sizing and cropping // strip everything after the "@." and add UY64 to limit the max image dimension to 640 int n = u.url().indexOf(QStringLiteral("@.")); if(n > -1) { const QString newLink = u.url().left(n) + QStringLiteral("@.UY640.jpg"); const QString id = ImageFactory::addImage(QUrl(newLink), true); if(!id.isEmpty()) { entry_->setField(cover, id); return; } } const QString id = ImageFactory::addImage(u, true); if(!id.isEmpty()) { entry_->setField(cover, id); return; } } } pos = linkRx.indexIn(str_, pos+linkRx.matchedLength()); } // <img alt="poster" posterRx.setPattern(QStringLiteral("<img\\s+[^>]*alt\\s*=\\s*\"poster\"[^>]+src\\s*=\\s*\"([^\"]+)\"")); pos = posterRx.indexIn(str_); if(pos > -1) { QUrl u = QUrl(baseURL_).resolved(QUrl(posterRx.cap(1))); QString id = ImageFactory::addImage(u, true); if(!id.isEmpty()) { entry_->setField(cover, id); return; } } // didn't find the cover, IMDb also used to put "cover" inside the url // cover is the img with the "cover" alt text pos = imgRx.indexIn(str_); while(pos > -1) { const QString url = imgRx.cap(0).toLower(); if(url.contains(cover)) { QUrl u = QUrl(baseURL_).resolved(QUrl(imgRx.cap(1))); QString id = ImageFactory::addImage(u, true); if(!id.isEmpty()) { entry_->setField(cover, id); return; } } pos = imgRx.indexIn(str_, pos+imgRx.matchedLength()); } } void IMDBFetcher::doLists2(const QString& str_, Tellico::Data::EntryPtr entry_) { QRegExp divInfoRx(QStringLiteral("<div class=\"info\">(.*)</div"), Qt::CaseInsensitive); divInfoRx.setMinimal(true); const LangData& data = langData(m_lang); QStringList genres, countries, langs, certs, tracks; for(int pos = divInfoRx.indexIn(str_); pos > -1; pos = divInfoRx.indexIn(str_, pos+divInfoRx.matchedLength())) { const QString text = divInfoRx.cap(1).remove(*s_tagRx); const QString tag = text.section(QLatin1Char(':'), 0, 0).simplified(); QString value = text.section(QLatin1Char(':'), 1, -1).simplified(); if(tag == data.genre) { foreach(const QString& token, value.split(QLatin1Char('|'))) { genres << token.trimmed(); } } else if(tag == data.language) { foreach(const QString& token, value.split(QRegExp(QLatin1String("[,|]")))) { langs << token.trimmed(); } } else if(tag == data.sound) { foreach(const QString& token, value.split(QLatin1Char('|'))) { tracks << token.trimmed(); } } else if(tag == data.country) { countries << value; } else if(tag == data.certification) { foreach(const QString& token, value.split(QLatin1Char('|'))) { certs << token.trimmed(); } } else if(tag == data.color) { // cut off any parentheses value = value.section(QLatin1Char('('), 0, 0).trimmed(); // change "black and white" to "black & white" value.replace(QStringLiteral("and"), QStringLiteral("&")); if(value == data.color) { entry_->setField(QStringLiteral("color"), i18n("Color")); } else { entry_->setField(QStringLiteral("color"), value); } } } entry_->setField(QStringLiteral("genre"), genres.join(FieldFormat::delimiterString())); entry_->setField(QStringLiteral("nationality"), countries.join(FieldFormat::delimiterString())); entry_->setField(QStringLiteral("language"), langs.join(FieldFormat::delimiterString())); entry_->setField(QStringLiteral("audio-track"), tracks.join(FieldFormat::delimiterString())); if(!certs.isEmpty()) { // first try to set default certification const QStringList& certsAllowed = entry_->collection()->fieldByName(QStringLiteral("certification"))->allowed(); foreach(const QString& cert, certs) { QString country = cert.section(QLatin1Char(':'), 0, 0); QString lcert = cert.section(QLatin1Char(':'), 1, 1); if(lcert == QStringLiteral("Unrated")) { lcert = QLatin1Char('U'); } lcert += QStringLiteral(" (") + country + QLatin1Char(')'); if(certsAllowed.contains(lcert)) { entry_->setField(QStringLiteral("certification"), lcert); break; } } // now add new field for all certifications const QString allc = QStringLiteral("allcertification"); if(optionalFields().contains(allc)) { Data::FieldPtr f = entry_->collection()->fieldByName(allc); if(!f) { f = new Data::Field(allc, i18n("Certifications"), Data::Field::Table); f->setFlags(Data::Field::AllowGrouped); entry_->collection()->addField(f); } entry_->setField(QStringLiteral("allcertification"), certs.join(FieldFormat::rowDelimiterString())); } } } // look at every anchor tag in the string void IMDBFetcher::doLists(const QString& str_, Tellico::Data::EntryPtr entry_) { const QString genre = QStringLiteral("/Genres/"); const QString genre2 = QStringLiteral("/genre/"); const QString country = QStringLiteral("/country/"); const QString lang = QStringLiteral("/language/"); const QString colorInfo = QStringLiteral("colors="); const QString cert = QStringLiteral("certificates="); const QString soundMix = QStringLiteral("sound_mixes="); const QString year = QStringLiteral("/Years/"); // if we reach faqs or user comments, we can stop const QString faqs = QStringLiteral("/faq"); const QString users = QStringLiteral("/user/"); // IMdb also has links with the word "sections" in them, remove that // for genres and nationalities int startPos = str_.indexOf(QStringLiteral("<div id=\"pagecontent\">")); if(startPos == -1) { startPos = 0; } QStringList genres, countries, langs, certs, tracks; for(int pos = s_anchorRx->indexIn(str_, startPos); pos > -1; pos = s_anchorRx->indexIn(str_, pos+s_anchorRx->matchedLength())) { const QString cap1 = s_anchorRx->cap(1); if(cap1.contains(genre) || cap1.contains(genre2)) { const QString g = s_anchorRx->cap(2); if(!g.contains(QStringLiteral(" section"), Qt::CaseInsensitive) && !g.contains(QStringLiteral(" genre"), Qt::CaseInsensitive)) { // ignore "Most Popular by Genre" genres += g.trimmed(); } } else if(cap1.contains(country)) { if(!s_anchorRx->cap(2).contains(QStringLiteral(" section"), Qt::CaseInsensitive)) { countries += s_anchorRx->cap(2).trimmed(); } } else if(cap1.contains(lang) && !cap1.contains(QStringLiteral("contribute"))) { langs += s_anchorRx->cap(2).trimmed(); } else if(cap1.contains(colorInfo)) { QString value = s_anchorRx->cap(2); // cut off any parentheses value = value.section(QLatin1Char('('), 0, 0).trimmed(); // change "black and white" to "black & white" value.replace(QStringLiteral("and"), QStringLiteral("&")); entry_->setField(QStringLiteral("color"), value.trimmed()); } else if(cap1.contains(cert)) { certs += s_anchorRx->cap(2).trimmed(); } else if(cap1.contains(soundMix)) { tracks += s_anchorRx->cap(2).trimmed(); // if year field wasn't set before, do it now } else if(entry_->field(QStringLiteral("year")).isEmpty() && cap1.contains(year)) { entry_->setField(QStringLiteral("year"), s_anchorRx->cap(2).trimmed()); } else if((cap1.contains(faqs) || cap1.contains(users)) && !genres.isEmpty()) { break; } } // since we have multiple genre search strings genres.removeDuplicates(); entry_->setField(QStringLiteral("genre"), genres.join(FieldFormat::delimiterString())); entry_->setField(QStringLiteral("nationality"), countries.join(FieldFormat::delimiterString())); entry_->setField(QStringLiteral("language"), langs.join(FieldFormat::delimiterString())); entry_->setField(QStringLiteral("audio-track"), tracks.join(FieldFormat::delimiterString())); if(!certs.isEmpty()) { // first try to set default certification const QStringList& certsAllowed = entry_->collection()->fieldByName(QStringLiteral("certification"))->allowed(); foreach(const QString& cert, certs) { QString country = cert.section(QLatin1Char(':'), 0, 0); if(country == QStringLiteral("United States")) { country = QStringLiteral("USA"); } QString lcert = cert.section(QLatin1Char(':'), 1, 1); if(lcert == QStringLiteral("Unrated")) { lcert = QLatin1Char('U'); } lcert += QStringLiteral(" (") + country + QLatin1Char(')'); if(certsAllowed.contains(lcert)) { entry_->setField(QStringLiteral("certification"), lcert); break; } } // now add new field for all certifications const QString allc = QStringLiteral("allcertification"); if(optionalFields().contains(allc)) { Data::FieldPtr f = entry_->collection()->fieldByName(allc); if(!f) { f = new Data::Field(allc, i18n("Certifications"), Data::Field::Table); f->setFlags(Data::Field::AllowGrouped); entry_->collection()->addField(f); } entry_->setField(QStringLiteral("allcertification"), certs.join(FieldFormat::rowDelimiterString())); } } } Tellico::Fetch::FetchRequest IMDBFetcher::updateRequest(Data::EntryPtr entry_) { const QString t = entry_->field(QStringLiteral("title")); QUrl link = QUrl::fromUserInput(entry_->field(QStringLiteral("imdb"))); if(!link.isEmpty() && link.isValid()) { if(link.host() != m_host) { // myLog() << "switching hosts to " << m_host; link.setHost(m_host); } return FetchRequest(Fetch::Raw, link.url()); } // optimistically try searching for title and rely on Collection::sameEntry() to figure things out if(!t.isEmpty()) { return FetchRequest(Fetch::Title, t); } return FetchRequest(); } QString IMDBFetcher::defaultName() { return i18n("Internet Movie Database"); } QString IMDBFetcher::defaultIcon() { return favIcon("https://www.imdb.com"); } //static Tellico::StringHash IMDBFetcher::allOptionalFields() { StringHash hash; hash[QStringLiteral("imdb")] = i18n("IMDb Link"); hash[QStringLiteral("imdb-rating")] = i18n("IMDb Rating"); hash[QStringLiteral("alttitle")] = i18n("Alternative Titles"); hash[QStringLiteral("allcertification")] = i18n("Certifications"); hash[QStringLiteral("origtitle")] = i18n("Original Title"); return hash; } Tellico::Fetch::ConfigWidget* IMDBFetcher::configWidget(QWidget* parent_) const { return new IMDBFetcher::ConfigWidget(parent_, this); } IMDBFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const IMDBFetcher* fetcher_/*=0*/) : Fetch::ConfigWidget(parent_) { QGridLayout* l = new QGridLayout(optionsWidget()); l->setSpacing(4); l->setColumnStretch(1, 10); int row = -1; /* IMDB.fr and others now redirects to imdb.com QLabel* label = new QLabel(i18n("Country: "), optionsWidget()); l->addWidget(label, ++row, 0); m_langCombo = new GUI::ComboBox(optionsWidget()); m_langCombo->addItem(i18n("United States"), EN); m_langCombo->addItem(i18n("France"), FR); m_langCombo->addItem(i18n("Spain"), ES); m_langCombo->addItem(i18n("Germany"), DE); m_langCombo->addItem(i18n("Italy"), IT); m_langCombo->addItem(i18n("Portugal"), PT); connect(m_langCombo, SIGNAL(activated(int)), SLOT(slotSetModified())); connect(m_langCombo, SIGNAL(activated(int)), SLOT(slotSiteChanged())); l->addWidget(m_langCombo, row, 1); QString w = i18n("The Internet Movie Database provides data from several different localized sites. " "Choose the one you wish to use for this data source."); label->setWhatsThis(w); m_langCombo->setWhatsThis(w); label->setBuddy(m_langCombo); */ QLabel* label = new QLabel(i18n("&Maximum cast: "), optionsWidget()); l->addWidget(label, ++row, 0); m_numCast = new QSpinBox(optionsWidget()); m_numCast->setMaximum(99); m_numCast->setMinimum(0); m_numCast->setValue(10); void (QSpinBox::* valueChanged)(const QString&) = &QSpinBox::valueChanged; connect(m_numCast, valueChanged, this, &ConfigWidget::slotSetModified); l->addWidget(m_numCast, row, 1); QString w = i18n("The list of cast members may include many people. Set the maximum number returned from the search."); label->setWhatsThis(w); m_numCast->setWhatsThis(w); label->setBuddy(m_numCast); m_fetchImageCheck = new QCheckBox(i18n("Download cover &image"), optionsWidget()); connect(m_fetchImageCheck, &QAbstractButton::clicked, this, &ConfigWidget::slotSetModified); ++row; l->addWidget(m_fetchImageCheck, row, 0, 1, 2); w = i18n("The cover image may be downloaded as well. However, too many large images in the " "collection may degrade performance."); m_fetchImageCheck->setWhatsThis(w); l->setRowStretch(++row, 10); // now add additional fields widget addFieldsWidget(IMDBFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList()); KAcceleratorManager::manage(optionsWidget()); if(fetcher_) { // m_langCombo->setCurrentData(fetcher_->m_lang); m_numCast->setValue(fetcher_->m_numCast); m_fetchImageCheck->setChecked(fetcher_->m_fetchImages); } else { //defaults // m_langCombo->setCurrentData(EN); m_numCast->setValue(10); m_fetchImageCheck->setChecked(true); } } void IMDBFetcher::ConfigWidget::saveConfigHook(KConfigGroup& config_) { // int n = m_langCombo->currentData().toInt(); // config_.writeEntry("Lang", n); config_.writeEntry("Host", QString()); // clear old host entry config_.writeEntry("Max Cast", m_numCast->value()); config_.writeEntry("Fetch Images", m_fetchImageCheck->isChecked()); } QString IMDBFetcher::ConfigWidget::preferredName() const { // return IMDBFetcher::langData(m_langCombo->currentData().toInt()).siteTitle; return IMDBFetcher::langData(EN).siteTitle; } void IMDBFetcher::ConfigWidget::slotSiteChanged() { emit signalName(preferredName()); } diff --git a/src/translators/csvimporter.cpp b/src/translators/csvimporter.cpp index 2cda7138..a4e43c2f 100644 --- a/src/translators/csvimporter.cpp +++ b/src/translators/csvimporter.cpp @@ -1,654 +1,654 @@ /*************************************************************************** Copyright (C) 2003-2009 Robby Stephenson <robby@periapsis.org> ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see <http://www.gnu.org/licenses/>. * * * ***************************************************************************/ #include "csvimporter.h" #include "csvparser.h" #include "translators.h" // needed for ImportAction #include "../collectionfieldsdialog.h" #include "../collection.h" #include "../tellico_debug.h" #include "../collectionfactory.h" #include "../gui/collectiontypecombo.h" #include "../utils/stringset.h" #include <KComboBox> #include <KSharedConfig> #include <KConfigGroup> #include <KMessageBox> #include <KLocalizedString> #include <QSpinBox> #include <QLineEdit> #include <QPushButton> #include <QGroupBox> #include <QLabel> #include <QCheckBox> #include <QRadioButton> #include <QTableWidget> #include <QHeaderView> #include <QRegExp> #include <QGridLayout> #include <QByteArray> #include <QVBoxLayout> #include <QHBoxLayout> #include <QButtonGroup> #include <QApplication> using Tellico::Import::CSVImporter; CSVImporter::CSVImporter(const QUrl& url_) : Tellico::Import::TextImporter(url_), m_existingCollection(nullptr), m_firstRowHeader(false), m_delimiter(QStringLiteral(",")), m_cancelled(false), m_widget(nullptr), m_comboColl(nullptr), m_checkFirstRowHeader(nullptr), m_radioComma(nullptr), m_radioSemicolon(nullptr), m_radioTab(nullptr), m_radioOther(nullptr), m_editOther(nullptr), m_editColDelimiter(nullptr), m_editRowDelimiter(nullptr), m_table(nullptr), m_colSpinBox(nullptr), m_comboField(nullptr), m_setColumnBtn(nullptr), m_hasAssignedFields(false), m_isLibraryThing(false), m_parser(new CSVParser(text())) { m_parser->setDelimiter(m_delimiter); } CSVImporter::~CSVImporter() { delete m_parser; m_parser = nullptr; } Tellico::Data::CollPtr CSVImporter::collection() { // don't just check if m_coll is non-null since the collection can be created elsewhere if(m_coll && m_coll->entryCount() > 0) { return m_coll; } if(!m_coll) { createCollection(); } const QStringList existingNames = m_coll->fieldNames(); QList<int> cols; cols.reserve(m_table->columnCount()); QStringList names; names.reserve(m_table->columnCount()); for(int col = 0; col < m_table->columnCount(); ++col) { QString t = m_table->horizontalHeaderItem(col)->text(); if(m_coll->fieldByTitle(t)) { cols << col; names << m_coll->fieldNameByTitle(t); } } if(names.isEmpty()) { myDebug() << "no fields assigned"; return Data::CollPtr(); } m_parser->reset(text()); // if the first row are headers, skip it if(m_firstRowHeader) { m_parser->skipLine(); } const uint numChars = text().size(); const uint stepSize = qMax(s_stepSize, numChars/100); const bool showProgress = options() & ImportProgress; // do we need to replace column or row delimiters const bool replaceColDelimiter = (!m_colDelimiter.isEmpty() && m_colDelimiter != FieldFormat::columnDelimiterString()); const bool replaceRowDelimiter = (!m_rowDelimiter.isEmpty() && m_rowDelimiter != FieldFormat::rowDelimiterString()); uint j = 0; while(!m_cancelled && m_parser->hasNext()) { bool empty = true; Data::EntryPtr entry(new Data::Entry(m_coll)); QStringList values = m_parser->nextTokens(); for(int i = 0; i < names.size(); ++i) { if(cols[i] >= values.size()) { break; } QString value = values[cols[i]].trimmed(); // only replace delimiters for tables // see https://forum.kde.org/viewtopic.php?f=200&t=142712 if(replaceColDelimiter && m_coll->fieldByName(names[i])->type() == Data::Field::Table) { value.replace(m_colDelimiter, FieldFormat::columnDelimiterString()); } if(replaceRowDelimiter && m_coll->fieldByName(names[i])->type() == Data::Field::Table) { value.replace(m_rowDelimiter, FieldFormat::rowDelimiterString()); } if(m_isLibraryThing) { // special cases for LibraryThing import if(names[i] == QLatin1String("isbn")) { // ISBN values are enclosed by brackets value.remove(QLatin1Char('[')).remove(QLatin1Char(']')); } else if(names[i] == QLatin1String("keyword")) { // LT values are comma-separated value.replace(QLatin1String(","), FieldFormat::delimiterString()); } else if(names[i] == QLatin1String("cdate")) { // only want date, not time. 10 characters since it's zero-padded value.truncate(10); } } bool success = entry->setField(names[i], value); // we might need to add a new allowed value // assume that if the user is importing the value, it should be allowed if(!success && m_coll->fieldByName(names[i])->type() == Data::Field::Choice) { Data::FieldPtr f = m_coll->fieldByName(names[i]); StringSet allow; allow.add(f->allowed()); allow.add(value); - f->setAllowed(allow.toList()); + f->setAllowed(allow.values()); m_coll->modifyField(f); success = entry->setField(names[i], value); } if(empty && success) { empty = false; } j += value.size(); } if(!empty) { m_coll->addEntries(entry); } if(showProgress && j%stepSize == 0) { emit signalProgress(this, 100*j/numChars); qApp->processEvents(); } } { KConfigGroup config(KSharedConfig::openConfig(), QStringLiteral("ImportOptions - CSV")); config.writeEntry("Delimiter", m_delimiter); config.writeEntry("ColumnDelimiter", m_colDelimiter); config.writeEntry("RowDelimiter", m_rowDelimiter); config.writeEntry("First Row Titles", m_firstRowHeader); } return m_coll; } QWidget* CSVImporter::widget(QWidget* parent_) { if(m_widget && m_widget->parent() == parent_) { return m_widget; } m_widget = new QWidget(parent_); QVBoxLayout* l = new QVBoxLayout(m_widget); QGroupBox* groupBox = new QGroupBox(i18n("CSV Options"), m_widget); QVBoxLayout* vlay = new QVBoxLayout(groupBox); QHBoxLayout* hlay = new QHBoxLayout(); vlay->addLayout(hlay); QLabel* lab = new QLabel(i18n("Collection &type:"), groupBox); hlay->addWidget(lab); m_comboColl = new GUI::CollectionTypeCombo(groupBox); hlay->addWidget(m_comboColl); lab->setBuddy(m_comboColl); m_comboColl->setWhatsThis(i18n("Select the type of collection being imported.")); void (QComboBox::* activatedInt)(int) = &QComboBox::activated; connect(m_comboColl, activatedInt, this, &CSVImporter::slotTypeChanged); m_checkFirstRowHeader = new QCheckBox(i18n("&First row contains field titles"), groupBox); m_checkFirstRowHeader->setWhatsThis(i18n("If checked, the first row is used as field titles.")); connect(m_checkFirstRowHeader, &QAbstractButton::toggled, this, &CSVImporter::slotFirstRowHeader); hlay->addWidget(m_checkFirstRowHeader); hlay->addStretch(10); QHBoxLayout* delimiterLayout = new QHBoxLayout(); vlay->addLayout(delimiterLayout); lab = new QLabel(i18n("Delimiter:"), groupBox); lab->setWhatsThis(i18n("In addition to a comma, other characters may be used as " "a delimiter, separating each value in the file.")); delimiterLayout->addWidget(lab); m_radioComma = new QRadioButton(groupBox); m_radioComma->setText(i18n("&Comma")); m_radioComma->setChecked(true); m_radioComma->setWhatsThis(i18n("Use a comma as the delimiter.")); delimiterLayout->addWidget(m_radioComma); m_radioSemicolon = new QRadioButton( groupBox); m_radioSemicolon->setText(i18n("&Semicolon")); m_radioSemicolon->setWhatsThis(i18n("Use a semi-colon as the delimiter.")); delimiterLayout->addWidget(m_radioSemicolon); m_radioTab = new QRadioButton(groupBox); m_radioTab->setText(i18n("Ta&b")); m_radioTab->setWhatsThis(i18n("Use a tab as the delimiter.")); delimiterLayout->addWidget(m_radioTab); m_radioOther = new QRadioButton(groupBox); m_radioOther->setText(i18n("Ot&her:")); m_radioOther->setWhatsThis(i18n("Use a custom string as the delimiter.")); delimiterLayout->addWidget(m_radioOther); m_editOther = new QLineEdit(groupBox); m_editOther->setEnabled(false); m_editOther->setFixedWidth(m_widget->fontMetrics().width(QLatin1Char('X')) * 4); m_editOther->setMaxLength(1); m_editOther->setWhatsThis(i18n("A custom string, such as a colon, may be used as a delimiter.")); m_editOther->setEnabled(false); delimiterLayout->addWidget(m_editOther); connect(m_radioOther, &QAbstractButton::toggled, m_editOther, &QWidget::setEnabled); connect(m_editOther, &QLineEdit::textChanged, this, &CSVImporter::slotDelimiter); delimiterLayout->addStretch(10); QButtonGroup* buttonGroup = new QButtonGroup(groupBox); buttonGroup->addButton(m_radioComma); buttonGroup->addButton(m_radioSemicolon); buttonGroup->addButton(m_radioTab); buttonGroup->addButton(m_radioOther); void (QButtonGroup::* buttonClickedInt)(int) = &QButtonGroup::buttonClicked; connect(buttonGroup, buttonClickedInt, this, &CSVImporter::slotDelimiter); QHBoxLayout* delimiterLayout2 = new QHBoxLayout(); vlay->addLayout(delimiterLayout2); QString w = i18n("The column delimiter separates values in each column of a <i>Table</i> field."); lab = new QLabel(i18n("Table column delimiter:"), groupBox); lab->setWhatsThis(w); delimiterLayout2->addWidget(lab); m_editColDelimiter = new QLineEdit(groupBox); m_editColDelimiter->setWhatsThis(w); m_editColDelimiter->setFixedWidth(m_widget->fontMetrics().width(QLatin1Char('X')) * 4); m_editColDelimiter->setMaxLength(1); delimiterLayout2->addWidget(m_editColDelimiter); connect(m_editColDelimiter, &QLineEdit::textChanged, this, &CSVImporter::slotDelimiter); w = i18n("The row delimiter separates values in each row of a <i>Table</i> field."); lab = new QLabel(i18n("Table row delimiter:"), groupBox); lab->setWhatsThis(w); delimiterLayout2->addWidget(lab); m_editRowDelimiter = new QLineEdit(groupBox); m_editRowDelimiter->setWhatsThis(w); m_editRowDelimiter->setFixedWidth(m_widget->fontMetrics().width(QLatin1Char('X')) * 4); m_editRowDelimiter->setMaxLength(1); delimiterLayout2->addWidget(m_editRowDelimiter); connect(m_editRowDelimiter, &QLineEdit::textChanged, this, &CSVImporter::slotDelimiter); delimiterLayout2->addStretch(10); m_table = new QTableWidget(5, 0, groupBox); vlay->addWidget(m_table); m_table->setSelectionMode(QAbstractItemView::SingleSelection); m_table->setSelectionBehavior(QAbstractItemView::SelectColumns); m_table->verticalHeader()->hide(); m_table->horizontalHeader()->setSectionsClickable(true); m_table->setMinimumHeight(m_widget->fontMetrics().lineSpacing() * 8); m_table->setWhatsThis(i18n("The table shows up to the first five lines of the CSV file.")); connect(m_table, &QTableWidget::currentCellChanged, this, &CSVImporter::slotCurrentChanged); connect(m_table->horizontalHeader(), &QHeaderView::sectionClicked, this, &CSVImporter::slotHeaderClicked); QHBoxLayout* hlay3 = new QHBoxLayout(); vlay->addLayout(hlay3); QString what = i18n("<qt>Set each column to correspond to a field in the collection by choosing " "a column, selecting the field, then clicking the <i>Assign Field</i> button.</qt>"); lab = new QLabel(i18n("Co&lumn:"), groupBox); hlay3->addWidget(lab); lab->setWhatsThis(what); m_colSpinBox = new QSpinBox(groupBox); hlay3->addWidget(m_colSpinBox); m_colSpinBox->setWhatsThis(what); m_colSpinBox->setMinimum(1); void (QSpinBox::* valueChangedInt)(int) = &QSpinBox::valueChanged; connect(m_colSpinBox, valueChangedInt, this, &CSVImporter::slotSelectColumn); lab->setBuddy(m_colSpinBox); hlay3->addSpacing(10); lab = new QLabel(i18n("&Data field in this column:"), groupBox); hlay3->addWidget(lab); lab->setWhatsThis(what); m_comboField = new KComboBox(groupBox); hlay3->addWidget(m_comboField); m_comboField->setWhatsThis(what); m_comboField->setFixedWidth(m_widget->fontMetrics().width(QLatin1Char('X')) * 20); // m_comboField->setSizeAdjustPolicy(QComboBox::AdjustToContents); connect(m_comboField, activatedInt, this, &CSVImporter::slotFieldChanged); lab->setBuddy(m_comboField); hlay3->addSpacing(10); m_setColumnBtn = new QPushButton(i18n("&Assign Field"), groupBox); hlay3->addWidget(m_setColumnBtn); m_setColumnBtn->setWhatsThis(what); m_setColumnBtn->setIcon(QIcon::fromTheme(QStringLiteral("dialog-ok-apply"))); connect(m_setColumnBtn, &QAbstractButton::clicked, this, &CSVImporter::slotSetColumnTitle); // hlay3->addStretch(10); l->addWidget(groupBox); l->addStretch(1); KConfigGroup config(KSharedConfig::openConfig(), QStringLiteral("ImportOptions - CSV")); m_delimiter = config.readEntry("Delimiter", m_delimiter); m_colDelimiter = config.readEntry("ColumnDelimiter", m_colDelimiter); m_rowDelimiter = config.readEntry("RowDelimiter", m_rowDelimiter); m_firstRowHeader = config.readEntry("First Row Titles", m_firstRowHeader); m_checkFirstRowHeader->setChecked(m_firstRowHeader); if(m_delimiter == QLatin1String(",")) { m_radioComma->setChecked(true); } else if(m_delimiter == QLatin1String(";")) { m_radioSemicolon->setChecked(true); } else if(m_delimiter == QLatin1String("\t")) { m_radioTab->setChecked(true); } else if(!m_delimiter.isEmpty()) { m_radioOther->setChecked(true); m_editOther->setEnabled(true); m_editOther->setText(m_delimiter); } slotDelimiter(); // initialize the parser and then load the text return m_widget; } bool CSVImporter::validImport() const { // at least one column has to be defined if(!m_hasAssignedFields) { KMessageBox::sorry(m_widget, i18n("At least one column must be assigned to a field. " "Only assigned columns will be imported.")); } return m_hasAssignedFields; } void CSVImporter::fillTable() { if(!m_table) { return; } m_parser->reset(text()); // not skipping first row since the updateHeader() call depends on it int maxCols = 0; int row = 0; for( ; m_parser->hasNext() && row < m_table->rowCount(); ++row) { QStringList values = m_parser->nextTokens(); if(static_cast<int>(values.count()) > m_table->columnCount()) { m_table->setColumnCount(values.count()); m_colSpinBox->setMaximum(values.count()); } int col = 0; foreach(const QString& value, values) { m_table->setItem(row, col, new QTableWidgetItem(value)); m_table->resizeColumnToContents(col); ++col; } if(col > maxCols) { maxCols = col; } // special case, check if the header row matches LibraryThing CSV export // assume LT export always uses identical header row and verify against first 7 columns if(row == 0 && values.count() > 7) { m_isLibraryThing = (values.at(0) == QLatin1String("'TITLE'") && values.at(1) == QLatin1String("'AUTHOR (first, last)'") && values.at(2) == QLatin1String("'AUTHOR (last, first)'") && values.at(3) == QLatin1String("'DATE'") && values.at(4) == QLatin1String("'LCC'") && values.at(5) == QLatin1String("'DDC'") && values.at(6) == QLatin1String("'ISBNs'")); } } for( ; row < m_table->rowCount(); ++row) { for(int col = 0; col < m_table->columnCount(); ++col) { delete m_table->takeItem(row, col); } } m_table->setColumnCount(maxCols); if(m_isLibraryThing) { // do not call slotFirstRowHeader since it will loop m_firstRowHeader = true; updateHeader(); } } void CSVImporter::slotTypeChanged() { createCollection(); updateHeader(); updateFieldCombo(); // hack to force a resize m_comboField->setFont(m_comboField->font()); m_comboField->updateGeometry(); } void CSVImporter::slotFirstRowHeader(bool b_) { m_firstRowHeader = b_; updateHeader(); fillTable(); } void CSVImporter::slotDelimiter() { if(m_radioComma->isChecked()) { m_delimiter = QStringLiteral(","); } else if(m_radioSemicolon->isChecked()) { m_delimiter = QStringLiteral(";"); } else if(m_radioTab->isChecked()) { m_delimiter = QStringLiteral("\t"); } else { m_editOther->setFocus(); m_delimiter = m_editOther->text(); } m_colDelimiter = m_editColDelimiter->text(); m_rowDelimiter = m_editRowDelimiter->text(); if(!m_delimiter.isEmpty()) { m_parser->setDelimiter(m_delimiter); fillTable(); updateHeader(); } } void CSVImporter::slotCurrentChanged(int, int col_) { const int pos = col_+1; m_colSpinBox->setValue(pos); //slotSelectColumn() gets called because of the signal } void CSVImporter::slotHeaderClicked(int col_) { const int pos = col_+1; m_colSpinBox->setValue(pos); //slotSelectColumn() gets called because of the signal } void CSVImporter::slotSelectColumn(int pos_) { // pos is really the number of the position of the column const int col = pos_ - 1; m_table->scrollToItem(m_table->item(0, col)); m_table->selectColumn(col); m_comboField->setCurrentItem(m_table->horizontalHeaderItem(col)->text()); } void CSVImporter::slotSetColumnTitle() { int col = m_colSpinBox->value()-1; const QString title = m_comboField->currentText(); m_table->horizontalHeaderItem(col)->setText(title); m_hasAssignedFields = true; // make sure none of the other columns have this title bool found = false; for(int i = 0; i < col; ++i) { if(m_table->horizontalHeaderItem(i)->text() == title) { m_table->horizontalHeaderItem(i)->setText(QString::number(i+1)); found = true; break; } } // if found, then we're done if(found) { return; } for(int i = col+1; i < m_table->columnCount(); ++i) { if(m_table->horizontalHeaderItem(i)->text() == title) { m_table->horizontalHeaderItem(i)->setText(QString::number(i+1)); break; } } } void CSVImporter::updateHeader() { if(!m_table) { return; } for(int col = 0; col < m_table->columnCount(); ++col) { QTableWidgetItem* headerItem = m_table->horizontalHeaderItem(col); if(!headerItem) { headerItem = new QTableWidgetItem(); m_table->setHorizontalHeaderItem(col, headerItem); } QTableWidgetItem* item = m_table->item(0, col); Data::FieldPtr field; if(item && m_coll) { QString itemValue = item->text(); // check against LibraryThing import if(m_isLibraryThing && m_coll->type() == Data::Collection::Book) { static QHash<QString, QString> ltFields; if(ltFields.isEmpty()) { ltFields[QStringLiteral("TITLE")] = QStringLiteral("title"); ltFields[QStringLiteral("AUTHOR (first, last)")] = QStringLiteral("author"); ltFields[QStringLiteral("DATE")] = QStringLiteral("pub_year"); ltFields[QStringLiteral("ISBNs")] = QStringLiteral("isbn"); ltFields[QStringLiteral("RATINGS")] = QStringLiteral("rating"); ltFields[QStringLiteral("ENTRY DATE")] = QStringLiteral("cdate"); ltFields[QStringLiteral("TAGS")] = QStringLiteral("keyword"); ltFields[QStringLiteral("COMMENT")] = QStringLiteral("comments"); ltFields[QStringLiteral("REVIEWS")] = QStringLiteral("review"); } // strip leading and trailing single quotes itemValue.remove(0,1).chop(1); itemValue = ltFields.value(itemValue); // review is a new field, we're going to add it by default if(itemValue == QLatin1String("review") && !m_coll->hasField(itemValue)) { Data::FieldPtr field(new Data::Field(QStringLiteral("review"), i18n("Review"), Data::Field::Para)); m_coll->addField(field); updateFieldCombo(); m_comboField->setCurrentIndex(m_comboField->count()-2); } } field = m_coll->fieldByTitle(itemValue); if(!field) { field = m_coll->fieldByName(itemValue); } } if(m_firstRowHeader && field) { headerItem->setText(field->title()); m_hasAssignedFields = true; } else { headerItem->setText(QString::number(col+1)); } } } void CSVImporter::slotFieldChanged(int idx_) { // only care if it's the last item -> add new field if(idx_ < m_comboField->count()-1) { return; } CollectionFieldsDialog dlg(m_coll, m_widget); dlg.setNotifyKernel(false); if(dlg.exec() == QDialog::Accepted) { updateFieldCombo(); fillTable(); } // set the combo to the item before last m_comboField->setCurrentIndex(m_comboField->count()-2); } void CSVImporter::slotActionChanged(int action_) { Data::CollPtr currColl = currentCollection(); if(!currColl) { m_existingCollection = nullptr; return; } switch(action_) { case Import::Replace: { int currType = m_comboColl->currentType(); m_comboColl->reset(); m_comboColl->setCurrentType(currType); m_existingCollection = nullptr; } break; case Import::Append: case Import::Merge: { m_comboColl->clear(); QString name = CollectionFactory::nameHash().value(currColl->type()); m_comboColl->addItem(name, currColl->type()); m_existingCollection = currColl; } break; } slotTypeChanged(); } void CSVImporter::slotCancel() { m_cancelled = true; } void CSVImporter::createCollection() { Data::Collection::Type type = static_cast<Data::Collection::Type>(m_comboColl->currentType()); m_coll = CollectionFactory::collection(type, true); if(m_existingCollection) { // if we're using the existing collection, then we // want the newly created collection to have the same fields foreach(Data::FieldPtr field, m_coll->fields()) { m_coll->removeField(field, true /* force */); } foreach(Data::FieldPtr field, m_existingCollection->fields()) { m_coll->addField(Data::FieldPtr(new Data::Field(*field))); } } } void CSVImporter::updateFieldCombo() { m_comboField->clear(); foreach(Data::FieldPtr field, m_coll->fields()) { m_comboField->addItem(field->title()); } m_comboField->addItem(i18n("<New Field>")); } diff --git a/src/translators/htmlexporter.cpp b/src/translators/htmlexporter.cpp index d5fc1460..6a941c0b 100644 --- a/src/translators/htmlexporter.cpp +++ b/src/translators/htmlexporter.cpp @@ -1,887 +1,887 @@ /*************************************************************************** Copyright (C) 2003-2009 Robby Stephenson <robby@periapsis.org> ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see <http://www.gnu.org/licenses/>. * * * ***************************************************************************/ #include "htmlexporter.h" #include "xslthandler.h" #include "tellicoxmlexporter.h" #include "../collection.h" #include "../document.h" #include "../core/filehandler.h" #include "../core/netaccess.h" #include "../config/tellico_config.h" #include "../core/tellico_strings.h" #include "../images/image.h" #include "../images/imagefactory.h" #include "../images/imageinfo.h" #include "../utils/tellico_utils.h" #include "../utils/string_utils.h" #include "../utils/datafileregistry.h" #include "../progressmanager.h" #include "../utils/cursorsaver.h" #include "../tellico_debug.h" #include <KConfigGroup> #include <KIO/MkdirJob> #include <KIO/FileCopyJob> #include <KIO/DeleteJob> #include <KLocalizedString> #include <KUser> #include <KJobWidgets> #include <QDir> #include <QDomDocument> #include <QGroupBox> #include <QCheckBox> #include <QFile> #include <QLabel> #include <QTextStream> #include <QVBoxLayout> #include <QFileInfo> #include <QApplication> #include <QLocale> extern "C" { #include <libxml/HTMLparser.h> #include <libxml/HTMLtree.h> } using Tellico::Export::HTMLExporter; HTMLExporter::HTMLExporter(Tellico::Data::CollPtr coll_) : Tellico::Export::Exporter(coll_), m_handler(nullptr), m_printHeaders(true), m_printGrouped(false), m_exportEntryFiles(false), m_cancelled(false), m_parseDOM(true), m_checkCreateDir(true), m_checkCommonFile(true), m_imageWidth(0), m_imageHeight(0), m_widget(nullptr), m_checkPrintHeaders(nullptr), m_checkPrintGrouped(nullptr), m_checkExportEntryFiles(nullptr), m_checkExportImages(nullptr), m_xsltFile(QStringLiteral("tellico2html.xsl")) { } HTMLExporter::~HTMLExporter() { delete m_handler; m_handler = nullptr; } QString HTMLExporter::formatString() const { return i18n("HTML"); } QString HTMLExporter::fileFilter() const { return i18n("HTML Files") + QLatin1String(" (*.html)") + QLatin1String(";;") + i18n("All Files") + QLatin1String(" (*)"); } void HTMLExporter::reset() { // since the ExportUTF8 option may have changed, need to delete handler delete m_handler; m_handler = nullptr; m_files.clear(); m_links.clear(); m_copiedFiles.clear(); } bool HTMLExporter::exec() { if(url().isEmpty() || !url().isValid()) { myWarning() << "trying to export to invalid URL"; return false; } // check file exists first // if we're not forcing, ask use bool force = (options() & Export::ExportForce) || FileHandler::queryExists(url()); if(!force) { return false; } m_cancelled = false; // TODO: maybe need label? if(options() & ExportProgress) { ProgressItem& item = ProgressManager::self()->newProgressItem(this, QString(), true); item.setTotalSteps(100); connect(&item, &Tellico::ProgressItem::signalCancelled, this, &Tellico::Export::HTMLExporter::slotCancel); } // ok if not ExportProgress, no worries ProgressItem::Done done(this); ProgressManager::self()->setProgress(this, 20); bool success = FileHandler::writeTextURL(url(), text(), options() & Export::ExportUTF8, force); if(m_parseDOM && !m_cancelled) { success &= copyFiles() && (!m_exportEntryFiles || writeEntryFiles()); } return success; } bool HTMLExporter::loadXSLTFile() { QString xsltFile = DataFileRegistry::self()->locate(m_xsltFile); if(xsltFile.isEmpty()) { myDebug() << "no xslt file for" << m_xsltFile; return false; } QUrl u = QUrl::fromLocalFile(xsltFile); // do NOT do namespace processing, it messes up the XSL declaration since // QDom thinks there are no elements in the Tellico namespace and as a result // removes the namespace declaration QDomDocument dom = FileHandler::readXMLDocument(u, false); if(dom.isNull()) { myDebug() << "error loading xslt file:" << xsltFile; return false; } // notes about utf-8 encoding: // all params should be passed to XSLTHandler in utf8 // input string to XSLTHandler should be in utf-8, EVEN IF DOM STRING SAYS OTHERWISE // the stylesheet prints utf-8 by default, if using locale encoding, need // to change the encoding attribute on the xsl:output element if(!(options() & Export::ExportUTF8)) { XSLTHandler::setLocaleEncoding(dom); } delete m_handler; m_handler = new XSLTHandler(dom, QFile::encodeName(xsltFile), true /*translate*/); if(m_checkCommonFile && !m_handler->isValid()) { Tellico::checkCommonXSLFile(); m_checkCommonFile = false; delete m_handler; m_handler = new XSLTHandler(dom, QFile::encodeName(xsltFile), true /*translate*/); } if(!m_handler->isValid()) { delete m_handler; m_handler = nullptr; return false; } m_handler->addStringParam("date", QDate::currentDate().toString(Qt::ISODate).toLatin1()); m_handler->addStringParam("time", QTime::currentTime().toString(Qt::ISODate).toLatin1()); m_handler->addStringParam("user", KUser(KUser::UseRealUserID).loginName().toLatin1()); if(m_exportEntryFiles) { // export entries to same place as all the other date files m_handler->addStringParam("entrydir", QFile::encodeName(fileDirName())); // be sure to link all the entries m_handler->addParam("link-entries", "true()"); } if(!m_collectionURL.isEmpty()) { QString s = QLatin1String("../") + m_collectionURL.fileName(); m_handler->addStringParam("collection-file", s.toUtf8()); } // look for a file that gets installed to know the installation directory // if parseDOM, that means we want the locations to be the actual location // otherwise, we assume it'll be relative if(m_parseDOM && m_dataDir.isEmpty()) { m_dataDir = Tellico::installationDir(); } else if(!m_parseDOM) { m_dataDir.clear(); } if(!m_dataDir.isEmpty()) { m_handler->addStringParam("datadir", QFile::encodeName(m_dataDir)); } setFormattingOptions(collection()); return m_handler->isValid(); } QString HTMLExporter::text() { if((!m_handler || !m_handler->isValid()) && !loadXSLTFile()) { myWarning() << "error loading xslt file:" << m_xsltFile; return QString(); } Data::CollPtr coll = collection(); if(!coll) { myDebug() << "no collection pointer!"; return QString(); } if(m_groupBy.isEmpty()) { m_printGrouped = false; // can't group if no groups exist } GUI::CursorSaver cs; writeImages(coll); // now grab the XML TellicoXMLExporter exporter(coll); exporter.setURL(url()); exporter.setEntries(entries()); exporter.setFields(fields()); exporter.setIncludeGroups(m_printGrouped); // yes, this should be in utf8, always exporter.setOptions(options() | Export::ExportUTF8 | Export::ExportImages); QDomDocument output = exporter.exportXML(); #if 0 QFile f(QLatin1String("/tmp/test.xml")); if(f.open(QIODevice::WriteOnly)) { QTextStream t(&f); t << output.toString(); } f.close(); #endif const QString outputText = m_handler->applyStylesheet(output.toString()); #if 0 myDebug() << "Remove debug2 from htmlexporter.cpp"; QFile f2(QLatin1String("/tmp/test.html")); if(f2.open(QIODevice::WriteOnly)) { QTextStream t(&f2); t << outputText; // t << "\n\n-------------------------------------------------------\n\n"; // t << Tellico::i18nReplace(outputText); } f2.close(); #endif if(!m_parseDOM) { return outputText; } htmlDocPtr htmlDoc = htmlParseDoc(reinterpret_cast<xmlChar*>(outputText.toUtf8().data()), nullptr); xmlNodePtr root = xmlDocGetRootElement(htmlDoc); if(root == nullptr) { myDebug() << "no root"; return outputText; } parseDOM(root); xmlChar* c; int bytes; htmlDocDumpMemory(htmlDoc, &c, &bytes); QString allText; if(bytes > 0) { allText = QString::fromUtf8(reinterpret_cast<const char*>(c), bytes); xmlFree(c); } return allText; } void HTMLExporter::setFormattingOptions(Tellico::Data::CollPtr coll) { QString file = Data::Document::self()->URL().fileName(); if(file != i18n(Tellico::untitledFilename)) { m_handler->addStringParam("filename", QFile::encodeName(file)); } m_handler->addStringParam("cdate", QLocale().toString(QDate::currentDate()).toUtf8()); m_handler->addParam("show-headers", m_printHeaders ? "true()" : "false()"); m_handler->addParam("group-entries", m_printGrouped ? "true()" : "false()"); QStringList sortTitles; if(!m_sort1.isEmpty()) { sortTitles << m_sort1; } if(!m_sort2.isEmpty()) { sortTitles << m_sort2; } // the third sort column may be same as first if(!m_sort3.isEmpty() && sortTitles.indexOf(m_sort3) == -1) { sortTitles << m_sort3; } if(sortTitles.count() > 0) { m_handler->addStringParam("sort-name1", coll->fieldNameByTitle(sortTitles[0]).toUtf8()); if(sortTitles.count() > 1) { m_handler->addStringParam("sort-name2", coll->fieldNameByTitle(sortTitles[1]).toUtf8()); if(sortTitles.count() > 2) { m_handler->addStringParam("sort-name3", coll->fieldNameByTitle(sortTitles[2]).toUtf8()); } } } // no longer showing "sorted by..." since the column headers are clickable // but still use "grouped by" QString sortString; if(m_printGrouped) { if(!m_groupBy.isEmpty()) { QString s; // if more than one, then it's the People pseudo-group if(m_groupBy.count() > 1) { s = i18n("People"); } else { s = coll->fieldTitleByName(m_groupBy[0]); } sortString = i18n("(grouped by %1)", s); } QString groupFields; for(QStringList::ConstIterator it = m_groupBy.constBegin(); it != m_groupBy.constEnd(); ++it) { Data::FieldPtr f = coll->fieldByName(*it); if(!f) { continue; } if(f->hasFlag(Data::Field::AllowMultiple)) { groupFields += QLatin1String("tc:") + *it + QLatin1String("s/tc:") + *it; } else { groupFields += QLatin1String("tc:") + *it; } int ncols = 0; if(f->type() == Data::Field::Table) { bool ok; ncols = Tellico::toUInt(f->property(QStringLiteral("columns")), &ok); if(!ok) { ncols = 1; } } if(ncols > 1) { groupFields += QLatin1String("/tc:column[1]"); } if(*it != m_groupBy.last()) { groupFields += QLatin1Char('|'); } } // myDebug() << groupFields; m_handler->addStringParam("group-fields", groupFields.toUtf8()); m_handler->addStringParam("sort-title", sortString.toUtf8()); } QString pageTitle = coll->title(); if(!sortString.isEmpty()) { pageTitle += QLatin1Char(' ') + sortString; } m_handler->addStringParam("page-title", pageTitle.toUtf8()); QStringList showFields; foreach(const QString& column, m_columns) { showFields << coll->fieldNameByTitle(column); } m_handler->addStringParam("column-names", showFields.join(QLatin1String(" ")).toUtf8()); if(m_imageWidth > 0 && m_imageHeight > 0) { m_handler->addParam("image-width", QByteArray().setNum(m_imageWidth)); m_handler->addParam("image-height", QByteArray().setNum(m_imageHeight)); } // add system colors to stylesheet const int type = coll->type(); m_handler->addStringParam("font", Config::templateFont(type).family().toLatin1()); m_handler->addStringParam("fontsize", QByteArray().setNum(Config::templateFont(type).pointSize())); m_handler->addStringParam("bgcolor", Config::templateBaseColor(type).name().toLatin1()); m_handler->addStringParam("fgcolor", Config::templateTextColor(type).name().toLatin1()); m_handler->addStringParam("color1", Config::templateHighlightedTextColor(type).name().toLatin1()); m_handler->addStringParam("color2", Config::templateHighlightedBaseColor(type).name().toLatin1()); // add locale code to stylesheet (for sorting) m_handler->addStringParam("lang", QLocale().name().toLatin1()); } void HTMLExporter::writeImages(Tellico::Data::CollPtr coll_) { // keep track of which image fields to write, this is for field titles StringSet imageFields; foreach(const QString& column, m_columns) { if(coll_->fieldByTitle(column) && coll_->fieldByTitle(column)->type() == Data::Field::Image) { imageFields.add(column); } } // all the images potentially used in the HTML export need to be written to disk // if we're exporting entry files, then we'll certainly want all the image fields written // if we're not exporting to a file, then we might be exporting an entry template file // and so we need to write all of them too. if(m_exportEntryFiles || url().isEmpty()) { // add all image fields to string list Data::FieldList iFields = coll_->imageFields(); // take intersection with the fields to be exported QSet<Data::FieldPtr> iFieldsSet = iFields.toSet(); - iFields = iFieldsSet.intersect(fields().toSet()).toList(); + iFields = iFieldsSet.intersect(fields().toSet()).values(); foreach(Data::FieldPtr field, iFields) { imageFields.add(field->name()); } } // all of them are going to get written to tmp file bool useTemp = url().isEmpty(); QUrl imgDir; QString imgDirRelative; // really some convoluted logic here // basically, four cases. 1) we're writing to a tmp file, for printing probably // so then write all the images to the tmp directory, 2) we're exporting to HTML, and // this is the main collection file, in which case m_parseDOM is always true; // 3) we're exporting HTML, and this is the first entry file, for which parseDOM is true // and exportEntryFiles is false. Then the image file will get copied in copyFiles() and is // probably an image in the entry template. 4) we're exporting HTML, and this is not the // first entry file, in which case, we want to refer directly to the target dir if(useTemp) { // everything goes in the tmp dir imgDir = QUrl::fromLocalFile(ImageFactory::tempDir()); imgDirRelative = imgDir.path(); } else if(m_parseDOM) { imgDir = fileDir(); // copy to fileDir imgDirRelative = ImageFactory::imageDir(); createDir(); } else { imgDir = fileDir(); imgDirRelative = QFileInfo(url().path()).dir().relativeFilePath(imgDir.path()); createDir(); } if(!imgDirRelative.endsWith(QLatin1Char('/'))) { imgDirRelative += QLatin1Char('/'); } m_handler->addStringParam("imgdir", QFile::encodeName(imgDirRelative)); int count = 0; const int processCount = 100; // process after every 100 events StringSet imageSet; // track which images are written foreach(const QString& imageField, imageFields) { foreach(Data::EntryPtr entryIt, entries()) { QString id = entryIt->field(imageField); // if no id or is already written, continue if(id.isEmpty() || imageSet.has(id)) { continue; } imageSet.add(id); // try writing bool success = false; if(useTemp) { // for link-only images, no need to write it out success = ImageFactory::imageInfo(id).linkOnly || ImageFactory::writeCachedImage(id, ImageFactory::TempDir); } else { const Data::Image& img = ImageFactory::imageById(id); QUrl target = imgDir; target = target.adjusted(QUrl::StripTrailingSlash); target.setPath(target.path() + QLatin1Char('/') + (id)); success = !img.isNull() && FileHandler::writeDataURL(target, img.byteArray(), true); } if(!success) { myWarning() << "unable to write image file: " << imgDir.path() << id; } if(++count == processCount) { qApp->processEvents(); count = 0; } } } } QWidget* HTMLExporter::widget(QWidget* parent_) { if(m_widget) { return m_widget; } m_widget = new QWidget(parent_); QVBoxLayout* l = new QVBoxLayout(m_widget); QGroupBox* gbox = new QGroupBox(i18n("HTML Options"), m_widget); QVBoxLayout* vlay = new QVBoxLayout(gbox); m_checkPrintHeaders = new QCheckBox(i18n("Print field headers"), gbox); m_checkPrintHeaders->setWhatsThis(i18n("If checked, the field names will be " "printed as table headers.")); m_checkPrintHeaders->setChecked(m_printHeaders); m_checkPrintGrouped = new QCheckBox(i18n("Group the entries"), gbox); m_checkPrintGrouped->setWhatsThis(i18n("If checked, the entries will be grouped by " "the selected field.")); m_checkPrintGrouped->setChecked(m_printGrouped); m_checkExportEntryFiles = new QCheckBox(i18n("Export individual entry files"), gbox); m_checkExportEntryFiles->setWhatsThis(i18n("If checked, individual files will be created for each entry.")); m_checkExportEntryFiles->setChecked(m_exportEntryFiles); vlay->addWidget(m_checkPrintHeaders); vlay->addWidget(m_checkPrintGrouped); vlay->addWidget(m_checkExportEntryFiles); l->addWidget(gbox); l->addStretch(1); return m_widget; } void HTMLExporter::readOptions(KSharedConfigPtr config_) { KConfigGroup exportConfig(config_, QStringLiteral("ExportOptions - %1").arg(formatString())); m_printHeaders = exportConfig.readEntry("Print Field Headers", m_printHeaders); m_printGrouped = exportConfig.readEntry("Print Grouped", m_printGrouped); m_exportEntryFiles = exportConfig.readEntry("Export Entry Files", m_exportEntryFiles); // read current entry export template m_entryXSLTFile = Config::templateName(collection()->type()); m_entryXSLTFile = DataFileRegistry::self()->locate(QLatin1String("entry-templates/") + m_entryXSLTFile + QLatin1String(".xsl")); } void HTMLExporter::saveOptions(KSharedConfigPtr config_) { KConfigGroup cfg(config_, QStringLiteral("ExportOptions - %1").arg(formatString())); m_printHeaders = m_checkPrintHeaders->isChecked(); cfg.writeEntry("Print Field Headers", m_printHeaders); m_printGrouped = m_checkPrintGrouped->isChecked(); cfg.writeEntry("Print Grouped", m_printGrouped); m_exportEntryFiles = m_checkExportEntryFiles->isChecked(); cfg.writeEntry("Export Entry Files", m_exportEntryFiles); } void HTMLExporter::setXSLTFile(const QString& filename_) { if(m_xsltFile == filename_) { return; } m_xsltFile = filename_; m_xsltFilePath.clear(); reset(); } void HTMLExporter::setEntryXSLTFile(const QString& fileName_) { QString fileName = fileName_; if(!fileName.endsWith(QLatin1String(".xsl"))) { fileName += QLatin1String(".xsl"); } QString f = DataFileRegistry::self()->locate(QLatin1String("entry-templates/") + fileName); if(f.isEmpty()) { myDebug() << fileName << "entry XSL file is not found"; } m_entryXSLTFile = f; } QUrl HTMLExporter::fileDir() const { if(url().isEmpty()) { return QUrl(); } QUrl fileDir = url(); // cd to directory of target URL fileDir = fileDir.adjusted(QUrl::RemoveFilename | QUrl::StripTrailingSlash); if(fileDirName().startsWith(QLatin1Char('/'))) { fileDir.setPath(fileDir.path() + fileDirName()); } else { fileDir.setPath(fileDir.path() + QLatin1Char('/') + fileDirName()); } return fileDir; } QString HTMLExporter::fileDirName() const { if(!m_collectionURL.isEmpty()) { return QStringLiteral("/"); } QFileInfo fi(url().fileName()); return fi.completeBaseName() + QLatin1String("_files/"); } // how ugly is this? const xmlChar* HTMLExporter::handleLink(const xmlChar* link_) { return reinterpret_cast<xmlChar*>(qstrdup(handleLink(QString::fromUtf8(reinterpret_cast<const char*>(link_))).toUtf8().constData())); } QString HTMLExporter::handleLink(const QString& link_) { if(link_.isEmpty()) { return link_; } if(m_links.contains(link_)) { return m_links[link_]; } const QUrl linkUrl(link_); // assume that if the link_ is not relative, then we don't need to copy it // also an invalid url is not relative either if(!linkUrl.isRelative()) { return link_; } if(m_xsltFilePath.isEmpty()) { m_xsltFilePath = DataFileRegistry::self()->locate(m_xsltFile); if(m_xsltFilePath.isEmpty()) { myWarning() << "no xslt file for " << m_xsltFile; } } QUrl u = QUrl::fromLocalFile(m_xsltFilePath); u = u.resolved(linkUrl); // one of the "quirks" of the html export is that img src urls are set to point to // the tmpDir() when exporting entry files from a collection, but those images // don't actually exist, and they get copied in writeImages() instead. // so we only need to keep track of the url if it exists const bool exists = NetAccess::exists(u, false, m_widget); if(exists) { m_files.append(u); } // if we're exporting entry files, we want pics/ to // go in pics/ const bool isPic = link_.startsWith(m_dataDir + QLatin1String("pics/")); QString midDir; if(m_exportEntryFiles && isPic) { midDir = QStringLiteral("pics/"); } // pictures are special since they might not exist when the HTML is exported, since they might get copied later // on the other hand, don't change the file location if it doesn't exist // and only use relative location if an export URL() is set if((isPic || exists) && !url().isEmpty()) { m_links.insert(link_, fileDirName() + midDir + u.fileName()); } else { m_links.insert(link_, link_); } // myDebug() << link_ << linkUrl << u << m_links[link_]; return m_links[link_]; } const xmlChar* HTMLExporter::analyzeInternalCSS(const xmlChar* str_) { return reinterpret_cast<xmlChar*>(qstrdup(analyzeInternalCSS(QString::fromUtf8(reinterpret_cast<const char*>(str_))).toUtf8().constData())); } QString HTMLExporter::analyzeInternalCSS(const QString& str_) { QString str = str_; int start = 0; int end = 0; const QString url = QStringLiteral("url("); for(int pos = str.indexOf(url); pos >= 0; pos = str.indexOf(url, pos+1)) { pos += 4; // url( if(str[pos] == QLatin1Char('"') || str[pos] == QLatin1Char('\'')) { ++pos; } start = pos; pos = str.indexOf(QLatin1Char(')'), start); end = pos; if(str[pos-1] == QLatin1Char('"') || str[pos-1] == QLatin1Char('\'')) { --end; } str.replace(start, end-start, handleLink(str.mid(start, end-start))); } return str; } void HTMLExporter::createDir() { if(!m_checkCreateDir) { return; } QUrl dir = fileDir(); if(dir.isEmpty()) { myDebug() << "called on empty URL!"; return; } if(NetAccess::exists(dir, false, m_widget)) { m_checkCreateDir = false; } else { KIO::Job* job = KIO::mkdir(dir); KJobWidgets::setWindow(job, m_widget); m_checkCreateDir = !job->exec(); } } bool HTMLExporter::copyFiles() { if(m_files.isEmpty()) { return true; } const int start = 20; const int maxProgress = m_exportEntryFiles ? 40 : 80; const int stepSize = qMax(1, m_files.count()/maxProgress); int j = 0; createDir(); QUrl target; for(QList<QUrl>::ConstIterator it = m_files.constBegin(); it != m_files.constEnd() && !m_cancelled; ++it, ++j) { if(m_copiedFiles.has((*it).url())) { continue; } if(target.isEmpty()) { target = fileDir(); } target = target.adjusted(QUrl::RemoveFilename); target.setPath(target.path() + (*it).fileName()); KIO::FileCopyJob* job = KIO::file_copy(*it, target, -1, KIO::Overwrite); KJobWidgets::setWindow(job, m_widget); if(job->exec()) { m_copiedFiles.add((*it).url()); } else { myWarning() << "can't copy " << target; myWarning() << job->errorString(); } if(j%stepSize == 0) { if(options() & ExportProgress) { ProgressManager::self()->setProgress(this, qMin(start+j/stepSize, 99)); } qApp->processEvents(); } } return true; } bool HTMLExporter::writeEntryFiles() { if(m_entryXSLTFile.isEmpty()) { myWarning() << "no entry XSLT file"; return false; } const int start = 60; const int stepSize = qMax(1, entries().count()/40); int j = 0; // now worry about actually exporting entry files // I can't reliable encode a string as a URI, so I'm punting, and I'll just replace everything but // a-zA-Z0-9 with an underscore. This MUST match the filename template in tellico2html.xsl // the id is used so uniqueness is guaranteed const QRegExp badChars(QLatin1String("[^-a-zA-Z0-9]")); FieldFormat::Request formatted = (options() & Export::ExportFormatted ? FieldFormat::ForceFormat : FieldFormat::AsIsFormat); QUrl outputFile = fileDir(); GUI::CursorSaver cs(Qt::WaitCursor); HTMLExporter exporter(collection()); long opt = options() | Export::ExportForce; opt &= ~ExportProgress; exporter.setFields(fields()); exporter.setOptions(opt); exporter.setXSLTFile(m_entryXSLTFile); exporter.setCollectionURL(url()); bool parseDOM = true; const QString title = QStringLiteral("title"); const QString html = QStringLiteral(".html"); bool multipleTitles = collection()->fieldByName(title)->hasFlag(Data::Field::AllowMultiple); Data::EntryList entries = this->entries(); // not const since the pointer has to be copied foreach(Data::EntryPtr entryIt, entries) { QString file = entryIt->formattedField(title, formatted); // but only use the first title if it has multiple if(multipleTitles) { file = file.section(QLatin1Char(';'), 0, 0); } file.replace(badChars, QStringLiteral("_")); file += QLatin1Char('-') + QString::number(entryIt->id()) + html; outputFile = outputFile.adjusted(QUrl::RemoveFilename); outputFile.setPath(outputFile.path() + file); exporter.setEntries(Data::EntryList() << entryIt); exporter.setURL(outputFile); exporter.exec(); // no longer need to parse DOM if(parseDOM) { parseDOM = false; exporter.setParseDOM(false); // this is rather stupid, but I'm too lazy to figure out the better way // since we parsed the DOM for the first entry file to grab any // images used in the template, need to resave it so the image links // get written correctly exporter.exec(); } if(j%stepSize == 0) { if(options() & ExportProgress) { ProgressManager::self()->setProgress(this, qMin(start+j/stepSize, 99)); } qApp->processEvents(); } ++j; } // the images in "pics/" are special data images, copy them always // since the entry files may refer to them, but we don't know that QStringList dataImages; dataImages.reserve(1 + 10); dataImages << QStringLiteral("checkmark.png"); for(uint i = 1; i <= 10; ++i) { dataImages << QStringLiteral("stars%1.png").arg(i); } QUrl dataDir = QUrl::fromLocalFile(Tellico::installationDir() + QLatin1String("pics/")); QUrl target = fileDir(); target = target.adjusted(QUrl::StripTrailingSlash); target.setPath(target.path() + QLatin1Char('/') + (QLatin1String("pics/"))); KIO::Job* job = KIO::mkdir(target); KJobWidgets::setWindow(job, m_widget); job->exec(); foreach(const QString& dataImage, dataImages) { dataDir = dataDir.adjusted(QUrl::RemoveFilename); dataDir.setPath(dataDir.path() + dataImage); target = target.adjusted(QUrl::RemoveFilename); target.setPath(target.path() + dataImage); KIO::Job* job = KIO::file_copy(dataDir, target); KJobWidgets::setWindow(job, m_widget); job->exec(); } return true; } void HTMLExporter::slotCancel() { m_cancelled = true; } void HTMLExporter::parseDOM(xmlNode* node_) { if(node_ == nullptr) { myDebug() << "no node"; return; } bool parseChildren = true; if(node_->type == XML_ELEMENT_NODE) { const QByteArray nodeName = QByteArray(reinterpret_cast<const char*>(node_->name)).toUpper(); xmlElement* elem = reinterpret_cast<xmlElement*>(node_); // to speed up things, check now for nodename if(nodeName == "IMG" || nodeName == "SCRIPT" || nodeName == "LINK") { for(xmlAttribute* attr = elem->attributes; attr; attr = reinterpret_cast<xmlAttribute*>(attr->next)) { QByteArray attrName = QByteArray(reinterpret_cast<const char*>(attr->name)).toUpper(); if( (attrName == "SRC" && (nodeName == "IMG" || nodeName == "SCRIPT")) || (attrName == "HREF" && nodeName == "LINK")) { /* (attrName == "BACKGROUND" && (nodeName == "BODY" || nodeName == "TABLE" || nodeName == "TH" || nodeName == "TD"))) */ xmlChar* value = xmlGetProp(node_, attr->name); if(value) { xmlSetProp(node_, attr->name, handleLink(value)); xmlFree(value); } // each node only has one significant attribute, so break now break; } } } else if(nodeName == "STYLE") { // if the first child is a CDATA, use it, otherwise replace complete node xmlNode* nodeToReplace = node_; xmlNode* child = node_->children; if(child && child->type == XML_CDATA_SECTION_NODE) { nodeToReplace = child; } xmlChar* value = xmlNodeGetContent(nodeToReplace); if(value) { xmlNodeSetContent(nodeToReplace, analyzeInternalCSS(value)); xmlFree(value); } // no longer need to parse child text nodes parseChildren = false; } } if(parseChildren) { xmlNode* child = node_->children; while(child) { parseDOM(child); child = child->next; } } } diff --git a/src/translators/tellicoxmlexporter.cpp b/src/translators/tellicoxmlexporter.cpp index 66a14d8d..a1c23285 100644 --- a/src/translators/tellicoxmlexporter.cpp +++ b/src/translators/tellicoxmlexporter.cpp @@ -1,598 +1,598 @@ /*************************************************************************** Copyright (C) 2003-2009 Robby Stephenson <robby@periapsis.org> ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see <http://www.gnu.org/licenses/>. * * * ***************************************************************************/ #include "tellicoxmlexporter.h" #include "tellico_xml.h" #include "../utils/bibtexhandler.h" // needed for cleaning text #include "../entrygroup.h" #include "../collections/bibtexcollection.h" #include "../images/imagefactory.h" #include "../images/image.h" #include "../images/imageinfo.h" #include "../core/filehandler.h" #include "../utils/string_utils.h" #include "../document.h" #include "../fieldformat.h" #include "../models/entrysortmodel.h" #include "../models/modelmanager.h" #include "../models/modeliterator.h" #include "../models/models.h" #include "../tellico_debug.h" #include <KLocalizedString> #include <KConfigGroup> #include <QDir> #include <QGroupBox> #include <QCheckBox> #include <QDomDocument> #include <QTextCodec> #include <QVBoxLayout> #include <algorithm> using namespace Tellico; using Tellico::Export::TellicoXMLExporter; TellicoXMLExporter::TellicoXMLExporter(Tellico::Data::CollPtr coll) : Exporter(coll), m_includeImages(false), m_includeGroups(false), m_widget(nullptr), m_checkIncludeImages(nullptr) { setOptions(options() | Export::ExportImages | Export::ExportImageSize); // not included by default } TellicoXMLExporter::~TellicoXMLExporter() { } QString TellicoXMLExporter::formatString() const { return i18n("XML"); } QString TellicoXMLExporter::fileFilter() const { return i18n("XML Files") + QLatin1String(" (*.xml)") + QLatin1String(";;") + i18n("All Files") + QLatin1String(" (*)"); } bool TellicoXMLExporter::exec() { QDomDocument doc = exportXML(); if(doc.isNull()) { return false; } return FileHandler::writeTextURL(url(), doc.toString(), options() & ExportUTF8, options() & Export::ExportForce); } QString TellicoXMLExporter::text() const { return exportXML().toString(); } QDomDocument TellicoXMLExporter::exportXML() const { int exportVersion = XML::syntaxVersion; if(exportVersion == 12 && !version12Needed()) { exportVersion = 11; } QDomImplementation impl; QDomDocumentType doctype = impl.createDocumentType(QStringLiteral("tellico"), XML::pubTellico(exportVersion), XML::dtdTellico(exportVersion)); //default namespace const QString& ns = XML::nsTellico; QDomDocument dom = impl.createDocument(ns, QStringLiteral("tellico"), doctype); // root tellico element QDomElement root = dom.documentElement(); QString encodeStr = QStringLiteral("version=\"1.0\" encoding=\""); if(options() & Export::ExportUTF8) { encodeStr += QLatin1String("UTF-8"); } else { encodeStr += QLatin1String(QTextCodec::codecForLocale()->name()); } encodeStr += QLatin1Char('"'); // createDocument creates a root node, insert the processing instruction before it dom.insertBefore(dom.createProcessingInstruction(QStringLiteral("xml"), encodeStr), root); root.setAttribute(QStringLiteral("syntaxVersion"), exportVersion); FieldFormat::Request format = (options() & Export::ExportFormatted ? FieldFormat::ForceFormat : FieldFormat::AsIsFormat); exportCollectionXML(dom, root, format); // clear image list m_images.clear(); return dom; } void TellicoXMLExporter::exportCollectionXML(QDomDocument& dom_, QDomElement& parent_, int format_) const { Data::CollPtr coll = collection(); if(!coll) { myWarning() << "no collection pointer!"; return; } QDomElement collElem = dom_.createElement(QStringLiteral("collection")); collElem.setAttribute(QStringLiteral("type"), coll->type()); collElem.setAttribute(QStringLiteral("title"), coll->title()); QDomElement fieldsElem = dom_.createElement(QStringLiteral("fields")); collElem.appendChild(fieldsElem); foreach(Data::FieldPtr field, fields()) { exportFieldXML(dom_, fieldsElem, field); } if(coll->type() == Data::Collection::Bibtex) { const Data::BibtexCollection* c = static_cast<const Data::BibtexCollection*>(coll.data()); if(!c->preamble().isEmpty()) { QDomElement preElem = dom_.createElement(QStringLiteral("bibtex-preamble")); preElem.appendChild(dom_.createTextNode(c->preamble())); collElem.appendChild(preElem); } QDomElement macrosElem = dom_.createElement(QStringLiteral("macros")); for(StringMap::ConstIterator macroIt = c->macroList().constBegin(); macroIt != c->macroList().constEnd(); ++macroIt) { if(!macroIt.value().isEmpty()) { QDomElement macroElem = dom_.createElement(QStringLiteral("macro")); macroElem.setAttribute(QStringLiteral("name"), macroIt.key()); macroElem.appendChild(dom_.createTextNode(macroIt.value())); macrosElem.appendChild(macroElem); } } if(macrosElem.childNodes().count() > 0) { collElem.appendChild(macrosElem); } } foreach(Data::EntryPtr entry, entries()) { exportEntryXML(dom_, collElem, entry, format_); } if(!m_images.isEmpty() && (options() & Export::ExportImages)) { QDomElement imgsElem = dom_.createElement(QStringLiteral("images")); - const QStringList imageIds = m_images.toList(); + const QStringList imageIds = m_images.values(); foreach(const QString& id, m_images) { exportImageXML(dom_, imgsElem, id); } if(imgsElem.hasChildNodes()) { collElem.appendChild(imgsElem); } } if(m_includeGroups) { exportGroupXML(dom_, collElem); } parent_.appendChild(collElem); // the borrowers and filters are in the tellico object, not the collection if(options() & Export::ExportComplete) { QDomElement bElem = dom_.createElement(QStringLiteral("borrowers")); foreach(Data::BorrowerPtr borrower, coll->borrowers()) { exportBorrowerXML(dom_, bElem, borrower); } if(bElem.hasChildNodes()) { parent_.appendChild(bElem); } QDomElement fElem = dom_.createElement(QStringLiteral("filters")); foreach(FilterPtr filter, coll->filters()) { exportFilterXML(dom_, fElem, filter); } if(fElem.hasChildNodes()) { parent_.appendChild(fElem); } } } void TellicoXMLExporter::exportFieldXML(QDomDocument& dom_, QDomElement& parent_, Tellico::Data::FieldPtr field_) const { QDomElement elem = dom_.createElement(QStringLiteral("field")); elem.setAttribute(QStringLiteral("name"), field_->name()); elem.setAttribute(QStringLiteral("title"), field_->title()); elem.setAttribute(QStringLiteral("category"), field_->category()); elem.setAttribute(QStringLiteral("type"), field_->type()); elem.setAttribute(QStringLiteral("flags"), field_->flags()); elem.setAttribute(QStringLiteral("format"), field_->formatType()); if(field_->type() == Data::Field::Choice) { elem.setAttribute(QStringLiteral("allowed"), field_->allowed().join(QLatin1String(";"))); } // only save description if it's not equal to title, which is the default // title is never empty, so this indirectly checks for empty descriptions if(field_->description() != field_->title()) { elem.setAttribute(QStringLiteral("description"), field_->description()); } for(StringMap::ConstIterator it = field_->propertyList().begin(); it != field_->propertyList().end(); ++it) { if(it.value().isEmpty()) { continue; } QDomElement e = dom_.createElement(QStringLiteral("prop")); e.setAttribute(QStringLiteral("name"), it.key()); e.appendChild(dom_.createTextNode(it.value())); elem.appendChild(e); } parent_.appendChild(elem); } void TellicoXMLExporter::exportEntryXML(QDomDocument& dom_, QDomElement& parent_, Tellico::Data::EntryPtr entry_, int format_) const { QDomElement entryElem = dom_.createElement(QStringLiteral("entry")); entryElem.setAttribute(QStringLiteral("id"), QString::number(entry_->id())); // iterate through every field for the entry foreach(Data::FieldPtr fIt, fields()) { QString fieldName = fIt->name(); // Date fields are special, don't format in export QString fieldValue = (format_ == FieldFormat::ForceFormat && fIt->type() != Data::Field::Date) ? entry_->formattedField(fieldName, FieldFormat::ForceFormat) : entry_->field(fieldName); if(options() & ExportClean) { BibtexHandler::cleanText(fieldValue); } // if empty, then no field element is added and just continue if(fieldValue.isEmpty()) { continue; } // optionally, verify images exist if(fIt->type() == Data::Field::Image && (options() & Export::ExportVerifyImages)) { if(!ImageFactory::validImage(fieldValue)) { myDebug() << "entry: " << entry_->title(); myDebug() << "skipping image: " << fieldValue; continue; } } if(fIt->type() == Data::Field::Table) { // who cares about grammar, just add an 's' to the name QDomElement parElem = dom_.createElement(fieldName + QLatin1Char('s')); entryElem.appendChild(parElem); bool ok; int ncols = Tellico::toUInt(fIt->property(QStringLiteral("columns")), &ok); if(!ok || ncols < 1) { ncols = 1; } foreach(const QString& rowValue, FieldFormat::splitTable(fieldValue)) { QDomElement fieldElem = dom_.createElement(fieldName); parElem.appendChild(fieldElem); QStringList columnValues = FieldFormat::splitRow(rowValue); if(ncols < columnValues.count()) { // need to combine all the last values, from ncols-1 to end QString lastValue = QStringList(columnValues.mid(ncols-1)).join(FieldFormat::columnDelimiterString()); columnValues = columnValues.mid(0, ncols); columnValues.replace(ncols-1, lastValue); } for(int col = 0; col < columnValues.count(); ++col) { QDomElement elem = dom_.createElement(QStringLiteral("column")); elem.appendChild(dom_.createTextNode(columnValues.at(col))); fieldElem.appendChild(elem); } } continue; } if(fIt->hasFlag(Data::Field::AllowMultiple)) { // if multiple versions are allowed, split them into separate elements // parent element if field contains multiple values, child of entryElem // who cares about grammar, just add an QLatin1Char('s') to the name QDomElement parElem = dom_.createElement(fieldName + QLatin1Char('s')); entryElem.appendChild(parElem); // the space after the semi-colon is enforced when the field is set for the entry QStringList fields = FieldFormat::splitValue(fieldValue); for(QStringList::ConstIterator it = fields.constBegin(); it != fields.constEnd(); ++it) { // element for field value, child of either entryElem or ParentElem QDomElement fieldElem = dom_.createElement(fieldName); fieldElem.appendChild(dom_.createTextNode(*it)); parElem.appendChild(fieldElem); } } else { QDomElement fieldElem = dom_.createElement(fieldName); entryElem.appendChild(fieldElem); // Date fields get special treatment if(fIt->type() == Data::Field::Date) { // as of Tellico in KF5 (3.0), just forget about the calendar attribute for the moment, always use gregorian fieldElem.setAttribute(QStringLiteral("calendar"), QStringLiteral("gregorian")); QStringList s = fieldValue.split(QLatin1Char('-'), QString::KeepEmptyParts); if(s.count() > 0 && !s[0].isEmpty()) { QDomElement e = dom_.createElement(QStringLiteral("year")); fieldElem.appendChild(e); e.appendChild(dom_.createTextNode(s[0])); } if(s.count() > 1 && !s[1].isEmpty()) { QDomElement e = dom_.createElement(QStringLiteral("month")); fieldElem.appendChild(e); e.appendChild(dom_.createTextNode(s[1])); } if(s.count() > 2 && !s[2].isEmpty()) { QDomElement e = dom_.createElement(QStringLiteral("day")); fieldElem.appendChild(e); e.appendChild(dom_.createTextNode(s[2])); } } else if(fIt->type() == Data::Field::URL && fIt->property(QStringLiteral("relative")) == QLatin1String("true") && !url().isEmpty()) { // if a relative URL and url() is not empty, change the value! QUrl old_url = Data::Document::self()->URL().resolved(QUrl(fieldValue)); QString relPath = QDir(url().toLocalFile()).relativeFilePath(old_url.path()); fieldElem.appendChild(dom_.createTextNode(relPath)); } else { fieldElem.appendChild(dom_.createTextNode(fieldValue)); } } if(fIt->type() == Data::Field::Image) { // possible to have more than one entry with the same image // only want to include it in the output xml once m_images.add(fieldValue); } } // end field loop parent_.appendChild(entryElem); } void TellicoXMLExporter::exportImageXML(QDomDocument& dom_, QDomElement& parent_, const QString& id_) const { if(id_.isEmpty()) { myDebug() << "empty image!"; return; } // myLog() << "id = " << id_; QDomElement imgElem = dom_.createElement(QStringLiteral("image")); if(m_includeImages) { const Data::Image& img = ImageFactory::imageById(id_); if(img.isNull()) { return; } imgElem.setAttribute(QStringLiteral("format"), QLatin1String(img.format())); imgElem.setAttribute(QStringLiteral("id"), QString(img.id())); imgElem.setAttribute(QStringLiteral("width"), img.width()); imgElem.setAttribute(QStringLiteral("height"), img.height()); if(img.linkOnly()) { imgElem.setAttribute(QStringLiteral("link"), QStringLiteral("true")); } QByteArray imgText = img.byteArray().toBase64(); imgElem.appendChild(dom_.createTextNode(QLatin1String(imgText))); } else { const Data::ImageInfo& info = ImageFactory::imageInfo(id_); if(info.isNull()) { return; } imgElem.setAttribute(QStringLiteral("format"), QLatin1String(info.format)); imgElem.setAttribute(QStringLiteral("id"), QString(info.id)); // only load the images to read the size if necessary const bool loadImageIfNecessary = options() & Export::ExportImageSize; imgElem.setAttribute(QStringLiteral("width"), info.width(loadImageIfNecessary)); imgElem.setAttribute(QStringLiteral("height"), info.height(loadImageIfNecessary)); if(info.linkOnly) { imgElem.setAttribute(QStringLiteral("link"), QStringLiteral("true")); } } parent_.appendChild(imgElem); } void TellicoXMLExporter::exportGroupXML(QDomDocument& dom_, QDomElement& parent_) const { Data::EntryList vec = entries(); bool exportAll = collection()->entries().count() == vec.count(); // iterate over each group, which are the first children for(ModelIterator gIt(ModelManager::self()->groupModel()); gIt.group(); ++gIt) { if(gIt.group()->isEmpty()) { continue; } QDomElement groupElem = dom_.createElement(QStringLiteral("group")); groupElem.setAttribute(QStringLiteral("title"), gIt.group()->groupName()); // now iterate over all entry items in the group Data::EntryList sorted = sortEntries(*gIt.group()); foreach(Data::EntryPtr eIt, sorted) { if(!exportAll && vec.indexOf(eIt) == -1) { continue; } QDomElement entryRefElem = dom_.createElement(QStringLiteral("entryRef")); entryRefElem.setAttribute(QStringLiteral("id"), QString::number(eIt->id())); groupElem.appendChild(entryRefElem); } if(groupElem.hasChildNodes()) { parent_.appendChild(groupElem); } } } void TellicoXMLExporter::exportFilterXML(QDomDocument& dom_, QDomElement& parent_, Tellico::FilterPtr filter_) const { QDomElement filterElem = dom_.createElement(QStringLiteral("filter")); filterElem.setAttribute(QStringLiteral("name"), filter_->name()); QString match = (filter_->op() == Filter::MatchAll) ? QStringLiteral("all") : QStringLiteral("any"); filterElem.setAttribute(QStringLiteral("match"), match); foreach(FilterRule* rule, *filter_) { QDomElement ruleElem = dom_.createElement(QStringLiteral("rule")); ruleElem.setAttribute(QStringLiteral("field"), rule->fieldName()); ruleElem.setAttribute(QStringLiteral("pattern"), rule->pattern()); switch(rule->function()) { case FilterRule::FuncContains: ruleElem.setAttribute(QStringLiteral("function"), QStringLiteral("contains")); break; case FilterRule::FuncNotContains: ruleElem.setAttribute(QStringLiteral("function"), QStringLiteral("notcontains")); break; case FilterRule::FuncEquals: ruleElem.setAttribute(QStringLiteral("function"), QStringLiteral("equals")); break; case FilterRule::FuncNotEquals: ruleElem.setAttribute(QStringLiteral("function"), QStringLiteral("notequals")); break; case FilterRule::FuncRegExp: ruleElem.setAttribute(QStringLiteral("function"), QStringLiteral("regexp")); break; case FilterRule::FuncNotRegExp: ruleElem.setAttribute(QStringLiteral("function"), QStringLiteral("notregexp")); break; case FilterRule::FuncBefore: ruleElem.setAttribute(QStringLiteral("function"), QStringLiteral("before")); break; case FilterRule::FuncAfter: ruleElem.setAttribute(QStringLiteral("function"), QStringLiteral("after")); break; case FilterRule::FuncGreater: ruleElem.setAttribute(QStringLiteral("function"), QStringLiteral("greaterthan")); break; case FilterRule::FuncLess: ruleElem.setAttribute(QStringLiteral("function"), QStringLiteral("lessthan")); break; /* If anything is updated here, be sure to update xmlstatehandler */ } filterElem.appendChild(ruleElem); } parent_.appendChild(filterElem); } void TellicoXMLExporter::exportBorrowerXML(QDomDocument& dom_, QDomElement& parent_, Tellico::Data::BorrowerPtr borrower_) const { if(borrower_->isEmpty()) { return; } QDomElement bElem = dom_.createElement(QStringLiteral("borrower")); parent_.appendChild(bElem); bElem.setAttribute(QStringLiteral("name"), borrower_->name()); bElem.setAttribute(QStringLiteral("uid"), borrower_->uid()); foreach(Data::LoanPtr it, borrower_->loans()) { QDomElement lElem = dom_.createElement(QStringLiteral("loan")); bElem.appendChild(lElem); lElem.setAttribute(QStringLiteral("uid"), it->uid()); lElem.setAttribute(QStringLiteral("entryRef"), QString::number(it->entry()->id())); lElem.setAttribute(QStringLiteral("loanDate"), it->loanDate().toString(Qt::ISODate)); lElem.setAttribute(QStringLiteral("dueDate"), it->dueDate().toString(Qt::ISODate)); if(it->inCalendar()) { lElem.setAttribute(QStringLiteral("calendar"), QStringLiteral("true")); } lElem.appendChild(dom_.createTextNode(it->note())); } } QWidget* TellicoXMLExporter::widget(QWidget* parent_) { if(m_widget) { return m_widget; } m_widget = new QWidget(parent_); QVBoxLayout* l = new QVBoxLayout(m_widget); QGroupBox* gbox = new QGroupBox(i18n("Tellico XML Options"), m_widget); QVBoxLayout* vlay = new QVBoxLayout(gbox); m_checkIncludeImages = new QCheckBox(i18n("Include images in XML document"), gbox); m_checkIncludeImages->setChecked(m_includeImages); m_checkIncludeImages->setWhatsThis(i18n("If checked, the images in the document will be included " "in the XML stream as base64 encoded elements.")); vlay->addWidget(m_checkIncludeImages); l->addWidget(gbox); l->addStretch(1); return m_widget; } void TellicoXMLExporter::readOptions(KSharedConfigPtr config_) { KConfigGroup group(config_, QStringLiteral("ExportOptions - %1").arg(formatString())); m_includeImages = group.readEntry("Include Images", m_includeImages); } void TellicoXMLExporter::saveOptions(KSharedConfigPtr config_) { m_includeImages = m_checkIncludeImages->isChecked(); KConfigGroup group(config_, QStringLiteral("ExportOptions - %1").arg(formatString())); group.writeEntry("Include Images", m_includeImages); } Tellico::Data::EntryList TellicoXMLExporter::sortEntries(const Data::EntryList& entries_) const { Data::EntryList sorted = entries_; EntrySortModel* model = static_cast<EntrySortModel*>(ModelManager::self()->entryModel()); // have to go in reverse for sorting Data::FieldList fields; Data::FieldPtr field; if(model->tertiarySortColumn() > -1) { field = model->headerData(model->tertiarySortColumn(), Qt::Horizontal, FieldPtrRole).value<Data::FieldPtr>(); if(field) { fields << field; } else { myDebug() << "no field for tertiary sort column" << model->tertiarySortColumn(); } } if(model->secondarySortColumn() > -1) { field = model->headerData(model->secondarySortColumn(), Qt::Horizontal, FieldPtrRole).value<Data::FieldPtr>(); if(field) { fields << field; } else { myDebug() << "no field for secondary sort column" << model->secondarySortColumn(); } } if(model->sortColumn() > -1) { field = model->headerData(model->sortColumn(), Qt::Horizontal, FieldPtrRole).value<Data::FieldPtr>(); if(field) { fields << field; } else { myDebug() << "no field for primary sort column" << model->sortColumn(); } } // now sort foreach(Data::FieldPtr field, fields) { std::sort(sorted.begin(), sorted.end(), Data::EntryCmp(field->name())); } return sorted; } bool TellicoXMLExporter::version12Needed() const { // version 12 is only necessary if the new filter rules are used foreach(FilterPtr filter, collection()->filters()) { foreach(FilterRule* rule, *filter) { if(rule->function() == FilterRule::FuncBefore || rule->function() == FilterRule::FuncAfter || rule->function() == FilterRule::FuncGreater || rule->function() == FilterRule::FuncLess) { return true; } } } return false; } diff --git a/src/translators/tellicozipexporter.cpp b/src/translators/tellicozipexporter.cpp index ce0967d1..1ae94d96 100644 --- a/src/translators/tellicozipexporter.cpp +++ b/src/translators/tellicozipexporter.cpp @@ -1,154 +1,154 @@ /*************************************************************************** Copyright (C) 2003-2009 Robby Stephenson <robby@periapsis.org> ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License or (at your option) version 3 or any later version * * accepted by the membership of KDE e.V. (or its successor approved * * by the membership of KDE e.V.), which shall act as a proxy * * defined in Section 14 of version 3 of the license. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see <http://www.gnu.org/licenses/>. * * * ***************************************************************************/ #include "tellicozipexporter.h" #include "tellicoxmlexporter.h" #include "../collection.h" #include "../images/imagefactory.h" #include "../images/image.h" #include "../images/imageinfo.h" #include "../core/filehandler.h" #include "../utils/stringset.h" #include "../tellico_debug.h" #include "../progressmanager.h" #include <KLocalizedString> #include <KZip> #include <QDomDocument> #include <QBuffer> #include <QApplication> using namespace Tellico; using Tellico::Export::TellicoZipExporter; TellicoZipExporter::TellicoZipExporter(Data::CollPtr coll) : Exporter(coll) , m_includeImages(true), m_cancelled(false) { } QString TellicoZipExporter::formatString() const { return i18n("Tellico Zip File"); } QString TellicoZipExporter::fileFilter() const { return i18n("Tellico Files") + QLatin1String(" (*.tc *.bc)") + QLatin1String(";;") + i18n("All Files") + QLatin1String(" (*)"); } bool TellicoZipExporter::exec() { m_cancelled = false; Data::CollPtr coll = collection(); if(!coll) { return false; } // TODO: maybe need label? ProgressItem& item = ProgressManager::self()->newProgressItem(this, QString(), true); item.setTotalSteps(100); connect(&item, &Tellico::ProgressItem::signalCancelled, this, &Tellico::Export::TellicoZipExporter::slotCancel); ProgressItem::Done done(this); TellicoXMLExporter exp(coll); exp.setEntries(entries()); exp.setFields(fields()); exp.setURL(url()); // needed in case of relative URL values long opt = options(); opt |= Export::ExportUTF8; // always export to UTF-8 opt |= Export::ExportImages; // always list the images in the xml opt &= ~Export::ExportProgress; // don't show progress for xml export exp.setOptions(opt); exp.setIncludeImages(false); // do not include the images themselves in XML QByteArray xml = exp.exportXML().toByteArray(); // encoded in utf-8 ProgressManager::self()->setProgress(this, 5); QByteArray data; QBuffer buf(&data); if(m_cancelled) { return true; // intentionally cancelled } KZip zip(&buf); zip.open(QIODevice::WriteOnly); zip.writeFile(QStringLiteral("tellico.xml"), xml); if(m_includeImages) { ProgressManager::self()->setProgress(this, 10); // gonna be lazy and just increment progress every 3 images // it might be less, might be more int j = 0; const QString imagesDir = QStringLiteral("images/"); StringSet imageSet; Data::FieldList imageFields = coll->imageFields(); // take intersection with the fields to be exported QSet<Data::FieldPtr> imageFieldSet = imageFields.toSet(); - imageFields = imageFieldSet.intersect(fields().toSet()).toList(); + imageFields = imageFieldSet.intersect(fields().toSet()).values(); // already took 10%, only 90% left const int stepSize = qMax(1, (coll->entryCount() * imageFields.count()) / 90); foreach(Data::EntryPtr entry, entries()) { if(m_cancelled) { break; } foreach(Data::FieldPtr imageField, imageFields) { const QString id = entry->field(imageField); if(id.isEmpty() || imageSet.has(id)) { continue; } const Data::ImageInfo& info = ImageFactory::imageInfo(id); if(info.linkOnly) { myLog() << "not copying linked image: " << id; continue; } const Data::Image& img = ImageFactory::imageById(id); // if no image, continue if(img.isNull()) { myWarning() << "no image found for " << imageField->title() << " field"; myWarning() << "...for the entry titled " << entry->title(); continue; } QByteArray ba = img.byteArray(); // myDebug() << "adding image id = " << it->field(fIt); zip.writeFile(imagesDir + id, ba); imageSet.add(id); if(j%stepSize == 0) { ProgressManager::self()->setProgress(this, qMin(10+j/stepSize, 99)); qApp->processEvents(); } ++j; } } } else { ProgressManager::self()->setProgress(this, 80); } zip.close(); if(m_cancelled) { return true; } return FileHandler::writeDataURL(url(), data, options() & Export::ExportForce); } void TellicoZipExporter::slotCancel() { m_cancelled = true; }