diff --git a/autotests/pkpassextractortest.cpp b/autotests/pkpassextractortest.cpp index fa43b13..e6c63ec 100644 --- a/autotests/pkpassextractortest.cpp +++ b/autotests/pkpassextractortest.cpp @@ -1,105 +1,106 @@ /* Copyright (C) 2018 Volker Krause This program is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace KItinerary; class PkPassExtractorTest : public QObject { Q_OBJECT private Q_SLOTS: void initTestCase() { // use some exotic locale to ensure the date/time parsing doesn't just work by luck QLocale::setDefault(QLocale(QStringLiteral("fr_FR"))); qputenv("TZ", "EST"); qRegisterMetaType(); } void testExtractText_data() { QTest::addColumn("inputFile"); QTest::addColumn("refFile"); QDir dir(QStringLiteral(SOURCE_DIR "/pkpassdata")); const auto lst = dir.entryList(QStringList(QStringLiteral("*.pkpass")), QDir::Files | QDir::Readable | QDir::NoSymLinks); for (const auto &file : lst) { const QString refFile = dir.path() + QLatin1Char('/') + file.left(file.size() - 7) + QStringLiteral(".json"); if (!QFile::exists(refFile)) { qDebug() << "reference file" << refFile << "does not exist, skipping test file" << file; continue; } QTest::newRow(file.toLatin1().constData()) << QString(dir.path() + QLatin1Char('/') + file) << refFile; } } void testExtractText() { QFETCH(QString, inputFile); QFETCH(QString, refFile); const auto pass = KPkPass::Pass::fromFile(inputFile, this); QVERIFY(pass); ExtractorRepository repo; const auto extractors = repo.extractorsForPass(pass); QVERIFY(!extractors.empty()); ExtractorEngine engine; engine.setSenderDate(QDateTime(QDate(2017, 12, 29), QTime(18, 46, 2))); engine.setExtractor(extractors.at(0)); engine.setPass(pass); auto result = JsonLdDocument::fromJson(engine.extract()); QCOMPARE(result.size(), 1); ExtractorPostprocessor postproc; + postproc.setContextDate(QDateTime(QDate(2017, 12, 29), QTime(18, 46, 2))); postproc.process(result); result = postproc.result(); QCOMPARE(result.size(), 1); const auto resJson = JsonLdDocument::toJson(result); QFile ref(refFile); QVERIFY(ref.open(QFile::ReadOnly)); const auto doc = QJsonDocument::fromJson(ref.readAll()); QVERIFY(doc.isArray()); if (resJson != doc.array()) { qDebug().noquote() << QJsonDocument(resJson).toJson(); } QCOMPARE(resJson, doc.array()); } }; QTEST_MAIN(PkPassExtractorTest) #include "pkpassextractortest.moc" diff --git a/autotests/postprocessordata/flight-filter.pre.json b/autotests/postprocessordata/flight-filter.pre.json index 3a60cde..c792f1f 100644 --- a/autotests/postprocessordata/flight-filter.pre.json +++ b/autotests/postprocessordata/flight-filter.pre.json @@ -1,73 +1,72 @@ [{ "@context": "http://schema.org", "@type": "FlightReservation", "reservationNumber": "RXJ34P", "reservationFor": { "@type": "Flight", "flightNumber": "110", "airline": { "@type": "Airline", "name": "United", "iataCode": "UA" }, "departureAirport": { "@type": "Airport", "name": "San Francisco Airport", "iataCode": "SFO" }, "departureTime": "2027-03-04T20:15:00-08:00", "arrivalTime": "2027-03-05T06:30:00-05:00" } }, { "@context": "http://schema.org", "@type": "FlightReservation", "reservationNumber": "XXX123" }, { "@context": "http://schema.org", "@type": "FlightReservation", "reservationNumber": "RXJ34P", "reservationFor": { "@type": "Flight", "flightNumber": "110", "airline": { "@type": "Airline", "name": "United", "iataCode": "UA" }, "departureAirport": { "@type": "Airport", "name": "San Francisco Airport", "iataCode": "SFO" }, "arrivalAirport": { "@type": "Airport", "name": "John F. Kennedy International Airport" }, "arrivalTime": "2027-03-05T06:30:00-05:00" } }, { "@context": "http://schema.org", "@type": "FlightReservation", "reservationNumber": "RXJ34P", "reservationFor": { "@type": "Flight", "flightNumber": "110", "airline": { "@type": "Airline", "name": "United", "iataCode": "UA" }, "departureAirport": { "@type": "Airport", "name": "San Francisco Airport", "iataCode": "SFO" }, "departureTime": "2027-03-04T20:15:00-08:00", "arrivalAirport": { - "@type": "Airport", - "name": "John F. Kennedy International Airport" + "@type": "Airport" } } } ] diff --git a/src/extractorpostprocessor.cpp b/src/extractorpostprocessor.cpp index 3f6f828..11a7f3b 100644 --- a/src/extractorpostprocessor.cpp +++ b/src/extractorpostprocessor.cpp @@ -1,294 +1,300 @@ /* Copyright (c) 2017 Volker Krause This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "extractorpostprocessor.h" #include "calendarhandler.h" #include "jsonlddocument.h" #include "airportdb/airportdb.h" #include "iatabcbpparser.h" #include #include #include #include #include #include #include #include #include using namespace KItinerary; namespace KItinerary { class ExtractorPostprocessorPrivate { public: QVariant processProperty(QVariant obj, const char *name, QVariant (ExtractorPostprocessorPrivate::*processor)(QVariant) const) const; QVariant processFlightReservation(QVariant res) const; QVariant processFlight(QVariant flight) const; QVariant processAirport(QVariant airport) const; QVariant processAirline(QVariant airline) const; void processFlightTime(QVariant &flight, const char *timePropName, const char *airportPropName) const; QVariant processReservation(QVariant res) const; bool filterReservation(const QVariant &res) const; bool filterLodgingReservation(const QVariant &res) const; bool filterFlight(const Flight &flight) const; bool filterAirport(const Airport &airport) const; bool filterTrainOrBusTrip(const QVariant &trip) const; bool filterTrainOrBusStation(const QVariant &station) const; QVector m_data; + QDateTime m_contextDate; }; } ExtractorPostprocessor::ExtractorPostprocessor() : d(new ExtractorPostprocessorPrivate) { } ExtractorPostprocessor::ExtractorPostprocessor(ExtractorPostprocessor &&) = default; ExtractorPostprocessor::~ExtractorPostprocessor() = default; void ExtractorPostprocessor::process(const QVector &data) { d->m_data.reserve(data.size()); for (auto elem : data) { if (elem.userType() == qMetaTypeId()) { elem = d->processFlightReservation(elem); } else if (elem.userType() == qMetaTypeId()) { elem = d->processReservation(elem); } else if (elem.userType() == qMetaTypeId()) { elem = d->processReservation(elem); } else if (elem.userType() == qMetaTypeId()) { elem = d->processReservation(elem); } if (d->filterReservation(elem)) { d->m_data.push_back(elem); } } std::stable_sort(d->m_data.begin(), d->m_data.end(), [](const QVariant &lhs, const QVariant &rhs) { return CalendarHandler::startDateTime(lhs) < CalendarHandler::startDateTime(rhs); }); } QVector ExtractorPostprocessor::result() const { return d->m_data; } +void ExtractorPostprocessor::setContextDate(const QDateTime& dt) +{ + d->m_contextDate = dt; +} + QVariant ExtractorPostprocessorPrivate::processProperty(QVariant obj, const char *name, QVariant (ExtractorPostprocessorPrivate::*processor)(QVariant) const) const { auto value = JsonLdDocument::readProperty(obj, name); value = (this->*processor)(value); JsonLdDocument::writeProperty(obj, name, value); return obj; } QVariant ExtractorPostprocessorPrivate::processFlightReservation(QVariant res) const { // expand ticketToken for IATA BCBP data auto bcbp = JsonLdDocument::readProperty(res, "ticketToken").toString(); if (!bcbp.isEmpty()) { if (bcbp.startsWith(QLatin1String("aztecCode:"))) { bcbp = bcbp.mid(10); } else if (bcbp.startsWith(QLatin1String("qrCode:"))) { bcbp = bcbp.mid(7); } - const auto bcbpData = IataBcbpParser::parse(bcbp); + const auto bcbpData = IataBcbpParser::parse(bcbp, m_contextDate.date()); if (bcbpData.size() == 1) { res = JsonLdDocument::apply(bcbpData.at(0), res); } } res = processReservation(res); res = processProperty(res, "reservationFor", &ExtractorPostprocessorPrivate::processFlight); return res; } QVariant ExtractorPostprocessorPrivate::processFlight(QVariant flight) const { flight = processProperty(flight, "departureAirport", &ExtractorPostprocessorPrivate::processAirport); flight = processProperty(flight, "arrivalAirport", &ExtractorPostprocessorPrivate::processAirport); flight = processProperty(flight, "airline", &ExtractorPostprocessorPrivate::processAirline); processFlightTime(flight, "boardingTime", "departureAirport"); processFlightTime(flight, "departureTime", "departureAirport"); processFlightTime(flight, "arrivalTime", "arrivalAirport"); return flight; } QVariant ExtractorPostprocessorPrivate::processAirport(QVariant airport) const { // clean up name const auto name = JsonLdDocument::readProperty(airport, "name").toString(); JsonLdDocument::writeProperty(airport, "name", name.trimmed()); // complete missing IATA codes auto iataCode = JsonLdDocument::readProperty(airport, "iataCode").toString(); if (iataCode.isEmpty()) { iataCode = AirportDb::iataCodeFromName(name).toString(); if (!iataCode.isEmpty()) { JsonLdDocument::writeProperty(airport, "iataCode", iataCode); } } // complete missing geo coordinates auto geo = JsonLdDocument::readProperty(airport, "geo"); if (!geo.value().isValid()) { const auto coord = AirportDb::coordinateForAirport(AirportDb::IataCode{iataCode}); if (coord.isValid()) { geo = QVariant::fromValue(GeoCoordinates()); JsonLdDocument::writeProperty(geo, "latitude", coord.latitude); JsonLdDocument::writeProperty(geo, "longitude", coord.longitude); JsonLdDocument::writeProperty(airport, "geo", geo); } } return airport; } QVariant ExtractorPostprocessorPrivate::processAirline(QVariant airline) const { const auto name = JsonLdDocument::readProperty(airline, "name").toString(); JsonLdDocument::writeProperty(airline, "name", name.trimmed()); return airline; } void ExtractorPostprocessorPrivate::processFlightTime(QVariant &flight, const char *timePropName, const char *airportPropName) const { const auto airport = JsonLdDocument::readProperty(flight, airportPropName); const auto iataCode = JsonLdDocument::readProperty(airport, "iataCode").toString(); if (iataCode.isEmpty()) { return; } auto dt = JsonLdDocument::readProperty(flight, timePropName).toDateTime(); if (!dt.isValid() || dt.timeSpec() == Qt::TimeZone) { return; } const auto tz = AirportDb::timezoneForAirport(AirportDb::IataCode{iataCode}); if (!tz.isValid()) { return; } // prefer our timezone over externally provided UTC offset, if they match if (dt.timeSpec() == Qt::OffsetFromUTC && tz.offsetFromUtc(dt) != dt.offsetFromUtc()) { return; } dt.setTimeSpec(Qt::TimeZone); dt.setTimeZone(tz); // if we updated from UTC offset to timezone spec here, QDateTime will compare equal // and the auto-generated property code will not actually update the property // so, clear the property first to force an update JsonLdDocument::writeProperty(flight, timePropName, QDateTime()); JsonLdDocument::writeProperty(flight, timePropName, dt); } QVariant ExtractorPostprocessorPrivate::processReservation(QVariant res) const { const auto viewUrl = JsonLdDocument::readProperty(res, "url").toUrl(); const auto modUrl = JsonLdDocument::readProperty(res, "modifyReservationUrl").toUrl(); const auto cancelUrl = JsonLdDocument::readProperty(res, "cancelReservationUrl").toUrl(); // remove duplicated urls if (modUrl.isValid() && viewUrl == modUrl) { JsonLdDocument::removeProperty(res, "modifyReservationUrl"); } if (cancelUrl.isValid() && viewUrl == cancelUrl) { JsonLdDocument::removeProperty(res, "cancelReservationUrl"); } // move ticketToken to Ticket (Google vs. schema.org difference) const auto token = JsonLdDocument::readProperty(res, "ticketToken").toString(); if (!token.isEmpty()) { auto ticket = JsonLdDocument::readProperty(res, "reservedTicket"); if (ticket.isNull()) { ticket = QVariant::fromValue(Ticket{}); } if (JsonLdDocument::readProperty(ticket, "ticketToken").toString().isEmpty()) { JsonLdDocument::writeProperty(ticket, "ticketToken", token); JsonLdDocument::writeProperty(res, "reservedTicket", ticket); } } return res; } bool ExtractorPostprocessorPrivate::filterReservation(const QVariant &res) const { const auto resFor = JsonLdDocument::readProperty(res, "reservationFor"); if (resFor.isNull()) { return false; } if (resFor.userType() == qMetaTypeId()) { return filterFlight(resFor.value()); } else if (resFor.userType() == qMetaTypeId()) { return filterTrainOrBusTrip(resFor); } else if (resFor.userType() == qMetaTypeId()) { return filterTrainOrBusTrip(resFor); } if (res.userType() == qMetaTypeId()) { return filterLodgingReservation(res); } return true; } bool ExtractorPostprocessorPrivate::filterLodgingReservation(const QVariant &res) const { const auto checkinDate = JsonLdDocument::readProperty(res, "checkinDate").toDateTime(); const auto checkoutDate = JsonLdDocument::readProperty(res, "checkoutDate").toDateTime(); return checkinDate.isValid() && checkoutDate.isValid(); } bool ExtractorPostprocessorPrivate::filterFlight(const Flight &flight) const { - const auto arrivalDepatureValid = flight.departureTime().isValid() && flight.arrivalTime().isValid(); - const auto boardingValid = flight.boardingTime().isValid(); + // this will be valid if either boarding time, departure time or departure day is set + const auto validDate = flight.departureDay().isValid(); return filterAirport(flight.departureAirport()) && filterAirport(flight.arrivalAirport()) - && (arrivalDepatureValid || boardingValid); + && validDate; } bool ExtractorPostprocessorPrivate::filterAirport(const Airport &airport) const { return !airport.iataCode().isEmpty() || !airport.name().isEmpty(); } bool ExtractorPostprocessorPrivate::filterTrainOrBusTrip(const QVariant &trip) const { const auto depDt = JsonLdDocument::readProperty(trip, "departureTime").toDateTime(); const auto arrDt = JsonLdDocument::readProperty(trip, "arrivalTime").toDateTime(); return filterTrainOrBusStation(JsonLdDocument::readProperty(trip, "departureStation")) && filterTrainOrBusStation(JsonLdDocument::readProperty(trip, "arrivalStation")) && depDt.isValid() && arrDt.isValid(); } bool ExtractorPostprocessorPrivate::filterTrainOrBusStation(const QVariant &station) const { return !JsonLdDocument::readProperty(station, "name").toString().isEmpty(); } diff --git a/src/extractorpostprocessor.h b/src/extractorpostprocessor.h index 19cd39a..5d02c84 100644 --- a/src/extractorpostprocessor.h +++ b/src/extractorpostprocessor.h @@ -1,52 +1,59 @@ /* Copyright (c) 2017 Volker Krause This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef EXTRACTORPOSTPROCESSOR_H #define EXTRACTORPOSTPROCESSOR_H #include "kitinerary_export.h" #include #include #include namespace KItinerary { class ExtractorPostprocessorPrivate; /** Post-process extracted data to filter out garbage and augment data from other sources. */ class KITINERARY_EXPORT ExtractorPostprocessor { public: ExtractorPostprocessor(); ExtractorPostprocessor(const ExtractorPostprocessor&) = delete; ExtractorPostprocessor(ExtractorPostprocessor&&); ~ExtractorPostprocessor(); void process(const QVector &data); QVector result() const; + /** The date the reservation(s) processed here have been made, if known. + * This is used for determining the year of incomplete dates provided by + * various sources. Therefore this has to be somewhen before the reservation + * becomes due. + */ + void setContextDate(const QDateTime &dt); + private: std::unique_ptr d; }; } #endif // EXTRACTORPOSTPROCESSOR_H