diff --git a/autotests/CMakeLists.txt b/autotests/CMakeLists.txt index ee81a3f..18cc3a6 100644 --- a/autotests/CMakeLists.txt +++ b/autotests/CMakeLists.txt @@ -1,27 +1,29 @@ add_definitions(-DSOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}") ecm_add_test(osmtypetest.cpp LINK_LIBRARIES Qt5::Test KOSM) +ecm_add_test(o5mparsertest.cpp LINK_LIBRARIES Qt5::Test KOSM) + ecm_add_test(indexeddatatabletest LINK_LIBRARIES Qt5::Test) ecm_add_test(mergeutiltest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(locationtest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(linetest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(departuretest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(journeytest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(platformtest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(notestest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(backendtest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(linemetadatatest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(navitiaparsertest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(hafasmgateparsertest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(hafasqueryparsertest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(efaparsertest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(deutschebahntest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(otpparsertest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(publictransportmanagertest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(cachetest.cpp LINK_LIBRARIES Qt5::Test KPublicTransport) ecm_add_test(mapviewtest.cpp LINK_LIBRARIES Qt5::Test KOSMIndoorMap) ecm_add_test(mapcssparsertest.cpp LINK_LIBRARIES Qt5::Test KOSMIndoorMap) ecm_add_test(scenegeometrytest.cpp LINK_LIBRARIES Qt5::Test KOSMIndoorMap) diff --git a/autotests/o5mparsertest.cpp b/autotests/o5mparsertest.cpp new file mode 100644 index 0000000..53bd311 --- /dev/null +++ b/autotests/o5mparsertest.cpp @@ -0,0 +1,83 @@ +/* + Copyright (C) 2020 Volker Krause + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU Library General Public License as published by + the Free Software Foundation; either version 2 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public + License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#include + +#include + +class O5mParserTest : public QObject +{ + Q_OBJECT +private Q_SLOTS: + void testParseUnsignedInt_data() + { + QTest::addColumn("data"); + QTest::addColumn("num"); + + QTest::newRow("0") << QByteArray::fromHex("00") << 0u; + QTest::newRow("1") << QByteArray::fromHex("01") << 1u; + QTest::newRow("127") << QByteArray::fromHex("7f") << 127u; + QTest::newRow("128") << QByteArray::fromHex("8001") << 128u; + QTest::newRow("255") << QByteArray::fromHex("FF01") << 255u; + QTest::newRow("256") << QByteArray::fromHex("8002") << 256u; + QTest::newRow("323") << QByteArray::fromHex("c302") << 323u; + QTest::newRow("16384") << QByteArray::fromHex("808001") << 16384u; + } + + void testParseUnsignedInt() + { + QFETCH(QByteArray, data); + QFETCH(uint32_t, num); + + OSM::O5mParser p(nullptr); + const auto beginIt = reinterpret_cast(data.constBegin()); + auto it = beginIt; + const auto endIt = reinterpret_cast(data.constEnd()); + QCOMPARE(p.readUnsigned(it, endIt), num); + QVERIFY(it > beginIt); + QVERIFY(it <= endIt); + } + + void testParseSignedInt_data() + { + QTest::addColumn("data"); + QTest::addColumn("num"); + + QTest::newRow("0") << QByteArray::fromHex("00") << 0; + QTest::newRow("64") << QByteArray::fromHex("8001") << 64; + QTest::newRow("-2") << QByteArray::fromHex("03") << -2; + QTest::newRow("-65") << QByteArray::fromHex("8101") << -65; + } + + void testParseSignedInt() + { + QFETCH(QByteArray, data); + QFETCH(int32_t, num); + + OSM::O5mParser p(nullptr); + const auto beginIt = reinterpret_cast(data.constBegin()); + auto it = beginIt; + const auto endIt = reinterpret_cast(data.constEnd()); + QCOMPARE(p.readSigned(it, endIt), num); + QVERIFY(it > beginIt); + QVERIFY(it <= endIt); + } +}; + +QTEST_GUILESS_MAIN(O5mParserTest) + +#include "o5mparsertest.moc" diff --git a/src/osm/CMakeLists.txt b/src/osm/CMakeLists.txt index 66c5b2a..a4c7b1c 100644 --- a/src/osm/CMakeLists.txt +++ b/src/osm/CMakeLists.txt @@ -1,12 +1,13 @@ add_library(KOSM STATIC datatypes.cpp element.cpp geomath.cpp + o5mparser.cpp overpassquery.cpp overpassquerymanager.cpp xmlparser.cpp ztile.cpp ) target_include_directories(KOSM PUBLIC "$") target_link_libraries(KOSM PUBLIC Qt5::Core PRIVATE Qt5::Network) diff --git a/src/osm/o5mparser.cpp b/src/osm/o5mparser.cpp new file mode 100644 index 0000000..d077fe3 --- /dev/null +++ b/src/osm/o5mparser.cpp @@ -0,0 +1,146 @@ +/* + Copyright (C) 2020 Volker Krause + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU Library General Public License as published by + the Free Software Foundation; either version 2 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public + License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#include "o5mparser.h" +#include "datatypes.h" + +#include + +#include + +using namespace OSM; + +enum : uint8_t { + O5M_RESET = 0xff, + O5M_NODE = 0x10, + O5M_WAY = 0x11, + O5M_RELATION = 0x12, + O5M_BOUNDING_BOX = 0xdb, + O5M_TIMESTAMP = 0xdc, + O5M_HEADER = 0xe0, + O5M_NUMBER_CONTINUATION = 0b1000'0000, + O5M_NUMBER_MASK = 0b0111'1111, + O5M_SIGNED_BIT = 0b1, +}; + +O5mParser::O5mParser(DataSet *dataSet) + : m_dataSet(dataSet) +{ +} + +void O5mParser::parse(const uint8_t* data, std::size_t len) +{ + qDebug() << "begin parsing"; + const auto endIt = data + len; + for (auto it = data; it < endIt - 1;) { + const auto blockType = (*it); + if (blockType == O5M_RESET) { + resetDeltaCodingState(); + ++it; + continue; + } + + auto blockSize = readUnsigned(++it, endIt); + if (blockSize >= (uint64_t)(endIt - it)) { + qWarning() << "premature end of file, or blocksize too large" << (endIt - it) << blockType << blockSize; + break; + } + switch (blockType) { + case O5M_HEADER: + if (blockSize != 4 || std::strncmp(reinterpret_cast(it), "o5m2", 4) != 0) { + qWarning() << "Invalid file header"; + return; + } + break; + case O5M_BOUNDING_BOX: + case O5M_TIMESTAMP: + // not of interest at the moment + break; + case O5M_NODE: + parseNode(it, it + blockSize); + break; + case O5M_WAY: + case O5M_RELATION: + // TODO + qDebug() << "todo:" << (it - data) << blockType << blockSize; + break; + default: + qDebug() << "unhandled o5m block type:" << (it - data) << blockType << blockSize; + } + + it += blockSize; + } + + qDebug() << "parsing done"; +} + +uint64_t O5mParser::readUnsigned(const uint8_t *&it, const uint8_t *endIt) const +{ + uint64_t result = 0; + int i = 0; + for (; it < endIt && ((*it) & O5M_NUMBER_CONTINUATION); ++it, ++i) { + result |= ((*it) & O5M_NUMBER_MASK) << (i * 7); + } + result |= ((*it++) & O5M_NUMBER_MASK) << (i * 7); + return result; +} + +int64_t O5mParser::readSigned(const uint8_t *&it, const uint8_t *endIt) const +{ + const auto u = readUnsigned(it, endIt); + return (u & O5M_SIGNED_BIT) ? (-(u >> 1) -1) : (u >> 1); +} + +template +T O5mParser::readDelta(const uint8_t *&it, const uint8_t *endIt, T &deltaState) +{ + deltaState += (T)readSigned(it, endIt); + return deltaState; +} + +void O5mParser::parseNode(const uint8_t *begin, const uint8_t *end) +{ + auto it = begin; + + const OSM::Id id = readDelta(it, end, m_nodeIdDelta); + if (it >= end) { return; } + + const auto version = readUnsigned(it, end); + if (version > 0) { + qWarning() << "skipping changeset data not implemented yet!"; + // timestamp (seconds since 1970, signed, delta-coded) + // author information – only if timestamp is not 0: + // changeset (signed, delta-coded) + // uid, user (string pair) + return; + } + if (it >= end) { return; } + + const auto lat = readDelta(it, end, m_latDelata); + const auto lon = readDelta(it, end, m_lonDelta); + qDebug() << " node " << id << (lat / 1.0e7) << (lon / 1.0e7); + if (it >= end) { return; } + + // TODO tags +} + +void O5mParser::resetDeltaCodingState() +{ + m_nodeIdDelta = 0; + m_latDelata = 0; + m_lonDelta = 0; +} diff --git a/src/osm/o5mparser.h b/src/osm/o5mparser.h new file mode 100644 index 0000000..957fbd9 --- /dev/null +++ b/src/osm/o5mparser.h @@ -0,0 +1,63 @@ +/* + Copyright (C) 2020 Volker Krause + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU Library General Public License as published by + the Free Software Foundation; either version 2 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public + License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#ifndef OSM_O5MPARSER_H +#define OSM_O5MPARSER_H + +#include + +class O5mParserTest; + +namespace OSM { + +class DataSet; + +/** Zero-copy parser of O5M binary files. + * @see https://wiki.openstreetmap.org/wiki/O5m + */ +class O5mParser +{ +public: + explicit O5mParser(DataSet *dataSet); + + /** Parse the given binary content. + * Feed this with QFile::map() for example. + */ + void parse(const uint8_t *data, std::size_t len); + +private: + friend class ::O5mParserTest; + + uint64_t readUnsigned(const uint8_t *&it, const uint8_t *endIt) const; + int64_t readSigned(const uint8_t *&it, const uint8_t *endIt) const; + template + T readDelta(const uint8_t *&it, const uint8_t *endIt, T &deltaState); + + void parseNode(const uint8_t *begin, const uint8_t *end); + + DataSet *m_dataSet = nullptr; + + // delta coding state + void resetDeltaCodingState(); + int64_t m_nodeIdDelta = 0; + int32_t m_latDelata = 0; // this can overflow, but that is intentional according to the spec! + int32_t m_lonDelta = 0; +}; + +} + +#endif // OSM_O5MPARSER_H