diff --git a/examples/dummyresource/CMakeLists.txt b/examples/dummyresource/CMakeLists.txt index 9f15e583..4c515c4f 100644 --- a/examples/dummyresource/CMakeLists.txt +++ b/examples/dummyresource/CMakeLists.txt @@ -1,10 +1,10 @@ project(sink_resource_dummy) include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) -add_library(${PROJECT_NAME} SHARED resourcefactory.cpp domainadaptor.cpp dummystore.cpp) +add_library(${PROJECT_NAME} SHARED resourcefactory.cpp dummystore.cpp) generate_flatbuffers(${PROJECT_NAME} dummycalendar) target_link_libraries(${PROJECT_NAME} sink Qt5::Core Qt5::Network) install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION ${SINK_RESOURCE_PLUGINS_PATH} RUNTIME DESTINATION ${SINK_RESOURCE_PLUGINS_PATH}) diff --git a/examples/dummyresource/domainadaptor.cpp b/examples/dummyresource/domainadaptor.cpp deleted file mode 100644 index e7a20da5..00000000 --- a/examples/dummyresource/domainadaptor.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2015 Christian Mollekopf - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the - * Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include "domainadaptor.h" - -#include "dummycalendar_generated.h" -#include "applicationdomaintype.h" - -using namespace DummyCalendar; -using namespace flatbuffers; - -DummyEventAdaptorFactory::DummyEventAdaptorFactory() - : DomainTypeAdaptorFactory() -{ -} - -DummyMailAdaptorFactory::DummyMailAdaptorFactory() - : DomainTypeAdaptorFactory() -{ - -} - -DummyFolderAdaptorFactory::DummyFolderAdaptorFactory() - : DomainTypeAdaptorFactory() -{ - -} - diff --git a/examples/dummyresource/domainadaptor.h b/examples/dummyresource/domainadaptor.h deleted file mode 100644 index 3faaa631..00000000 --- a/examples/dummyresource/domainadaptor.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2015 Christian Mollekopf - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the - * Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ -#pragma once - -#include "common/domainadaptor.h" -#include "event_generated.h" -#include "mail_generated.h" -#include "folder_generated.h" -#include "entity_generated.h" - -class DummyEventAdaptorFactory : public DomainTypeAdaptorFactory -{ -public: - DummyEventAdaptorFactory(); - virtual ~DummyEventAdaptorFactory() {}; -}; - -class DummyMailAdaptorFactory : public DomainTypeAdaptorFactory -{ -public: - DummyMailAdaptorFactory(); - virtual ~DummyMailAdaptorFactory() {}; -}; - -class DummyFolderAdaptorFactory : public DomainTypeAdaptorFactory -{ -public: - DummyFolderAdaptorFactory(); - virtual ~DummyFolderAdaptorFactory() {}; -}; diff --git a/examples/dummyresource/resourcefactory.cpp b/examples/dummyresource/resourcefactory.cpp index 12e0e1ea..d89ddbd2 100644 --- a/examples/dummyresource/resourcefactory.cpp +++ b/examples/dummyresource/resourcefactory.cpp @@ -1,210 +1,213 @@ /* * Copyright (C) 2014 Aaron Seigo * Copyright (C) 2015 Christian Mollekopf * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the * Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #include "resourcefactory.h" #include "facade.h" #include "entitybuffer.h" #include "pipeline.h" #include "dummycalendar_generated.h" #include "mail_generated.h" #include "domainadaptor.h" #include "log.h" #include "dummystore.h" #include "definitions.h" #include "facadefactory.h" #include "adaptorfactoryregistry.h" #include "synchronizer.h" #include "inspector.h" #include "mailpreprocessor.h" #include "eventpreprocessor.h" #include "specialpurposepreprocessor.h" #include //This is the resources entity type, and not the domain type #define ENTITY_TYPE_EVENT "event" #define ENTITY_TYPE_MAIL "mail" #define ENTITY_TYPE_FOLDER "folder" using namespace Sink; class DummySynchronizer : public Sink::Synchronizer { public: DummySynchronizer(const Sink::ResourceContext &context) : Sink::Synchronizer(context) { setSecret("dummy"); } Sink::ApplicationDomain::Event::Ptr createEvent(const QByteArray &ridBuffer, const QMap &data) { auto event = Sink::ApplicationDomain::Event::Ptr::create(); event->setExtractedUid(data.value("uid").toString()); event->setExtractedSummary(data.value("summary").toString()); event->setExtractedDescription(data.value("description").toString()); event->setExtractedStartTime(data.value("starttime").toDateTime()); event->setExtractedEndTime(data.value("endtime").toDateTime()); event->setProperty("remoteId", ridBuffer); return event; } Sink::ApplicationDomain::Mail::Ptr createMail(const QByteArray &ridBuffer, const QMap &data) { auto mail = Sink::ApplicationDomain::Mail::Ptr::create(); mail->setExtractedMessageId(ridBuffer); mail->setExtractedSubject(data.value("subject").toString()); mail->setExtractedSender(Sink::ApplicationDomain::Mail::Contact{data.value("senderName").toString(), data.value("senderEmail").toString()}); mail->setExtractedDate(data.value("date").toDateTime()); mail->setFolder(syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, data.value("parentFolder").toByteArray())); mail->setUnread(data.value("unread").toBool()); mail->setImportant(data.value("important").toBool()); return mail; } Sink::ApplicationDomain::Folder::Ptr createFolder(const QByteArray &ridBuffer, const QMap &data) { auto folder = Sink::ApplicationDomain::Folder::Ptr::create(); folder->setName(data.value("name").toString()); folder->setIcon(data.value("icon").toByteArray()); if (!data.value("parent").toString().isEmpty()) { auto sinkId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, data.value("parent").toByteArray()); folder->setParent(sinkId); } return folder; } void synchronize(const QByteArray &bufferType, const QMap > &data, std::function &data)> createEntity) { auto time = QSharedPointer::create(); time->start(); //TODO find items to remove int count = 0; for (auto it = data.constBegin(); it != data.constEnd(); it++) { count++; const auto remoteId = it.key().toUtf8(); auto entity = createEntity(remoteId, it.value()); createOrModify(bufferType, remoteId, *entity); } SinkTrace() << "Sync of " << count << " entities of type " << bufferType << " done." << Sink::Log::TraceTime(time->elapsed()); } KAsync::Job synchronizeWithSource(const Sink::QueryBase &) Q_DECL_OVERRIDE { SinkLog() << " Synchronizing with the source"; SinkTrace() << "Synchronize with source and sending a notification about it"; Sink::Notification n; n.id = "connected"; n.type = Sink::Notification::Status; n.message = "We're connected"; n.code = Sink::ApplicationDomain::ConnectedStatus; emit notify(n); return KAsync::start([this]() { synchronize(ENTITY_TYPE_EVENT, DummyStore::instance().events(), [this](const QByteArray &ridBuffer, const QMap &data) { return createEvent(ridBuffer, data); }); synchronize(ENTITY_TYPE_MAIL, DummyStore::instance().mails(), [this](const QByteArray &ridBuffer, const QMap &data) { return createMail(ridBuffer, data); }); synchronize(ENTITY_TYPE_FOLDER, DummyStore::instance().folders(), [this](const QByteArray &ridBuffer, const QMap &data) { return createFolder(ridBuffer, data); }); }); } bool canReplay(const QByteArray &type, const QByteArray &key, const QByteArray &value) Q_DECL_OVERRIDE { return false; } }; class DummyInspector : public Sink::Inspector { public: DummyInspector(const Sink::ResourceContext &resourceContext) : Sink::Inspector(resourceContext) { } protected: KAsync::Job inspect(int inspectionType, const QByteArray &inspectionId, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expectedValue) Q_DECL_OVERRIDE { SinkTrace() << "Inspecting " << inspectionType << domainType << entityId << property << expectedValue; if (property == "testInspection") { if (expectedValue.toBool()) { //Success return KAsync::null(); } else { //Failure return KAsync::error(1, "Failed."); } } return KAsync::null(); } }; DummyResource::DummyResource(const Sink::ResourceContext &resourceContext, const QSharedPointer &pipeline) : Sink::GenericResource(resourceContext, pipeline) { setupSynchronizer(QSharedPointer::create(resourceContext)); setupInspector(QSharedPointer::create(resourceContext)); setupPreprocessors(ENTITY_TYPE_MAIL, {new MailPropertyExtractor, new SpecialPurposeProcessor}); setupPreprocessors(ENTITY_TYPE_FOLDER, {}); setupPreprocessors(ENTITY_TYPE_EVENT, {new EventPropertyExtractor}); } DummyResource::~DummyResource() { } DummyResourceFactory::DummyResourceFactory(QObject *parent) : Sink::ResourceFactory(parent, {Sink::ApplicationDomain::ResourceCapabilities::Mail::mail, - "event", + Sink::ApplicationDomain::ResourceCapabilities::Event::event, + Sink::ApplicationDomain::ResourceCapabilities::Event::calendar, Sink::ApplicationDomain::ResourceCapabilities::Mail::folder, Sink::ApplicationDomain::ResourceCapabilities::Mail::storage, "-folder.rename", Sink::ApplicationDomain::ResourceCapabilities::Mail::sent} ) { } Sink::Resource *DummyResourceFactory::createResource(const Sink::ResourceContext &resourceContext) { return new DummyResource(resourceContext); } void DummyResourceFactory::registerFacades(const QByteArray &resourceName, Sink::FacadeFactory &factory) { factory.registerFacade>(resourceName); + factory.registerFacade>(resourceName); factory.registerFacade>(resourceName); factory.registerFacade>(resourceName); } void DummyResourceFactory::registerAdaptorFactories(const QByteArray &resourceName, Sink::AdaptorFactoryRegistry ®istry) { - registry.registerFactory(resourceName); - registry.registerFactory(resourceName); - registry.registerFactory(resourceName); + registry.registerFactory>(resourceName); + registry.registerFactory>(resourceName); + registry.registerFactory>(resourceName); + registry.registerFactory>(resourceName); } void DummyResourceFactory::removeDataFromDisk(const QByteArray &instanceIdentifier) { DummyResource::removeFromDisk(instanceIdentifier); } diff --git a/tests/dummyresourcebenchmark.cpp b/tests/dummyresourcebenchmark.cpp index 57ad4de8..ce09c8fd 100644 --- a/tests/dummyresourcebenchmark.cpp +++ b/tests/dummyresourcebenchmark.cpp @@ -1,157 +1,156 @@ #include #include #include "dummyresource/resourcefactory.h" -#include "dummyresource/domainadaptor.h" #include "store.h" #include "notifier.h" #include "resourcecontrol.h" #include "commands.h" #include "entitybuffer.h" #include "log.h" #include "resourceconfig.h" #include "notification_generated.h" #include "test.h" #include "testutils.h" #include "adaptorfactoryregistry.h" #include "hawd/dataset.h" #include "hawd/formatter.h" #include "event_generated.h" #include "entity_generated.h" #include "metadata_generated.h" #include "createentity_generated.h" /** * Benchmark full system with the dummy resource implementation. */ class DummyResourceBenchmark : public QObject { Q_OBJECT private: int num; private slots: void initTestCase() { Sink::Log::setDebugOutputLevel(Sink::Log::Warning); auto factory = Sink::ResourceFactory::load("sink.dummy"); QVERIFY(factory); ResourceConfig::addResource("sink.dummy.instance1", "sink.dummy"); num = 5000; } void cleanup() { } // Ensure we can process a command in less than 0.1s void testCommandResponsiveness() { // Test responsiveness including starting the process. VERIFYEXEC(Sink::Store::removeDataFromDisk("sink.dummy.instance1")); QTime time; time.start(); Sink::ApplicationDomain::Event event("sink.dummy.instance1"); event.setProperty("uid", "testuid"); QCOMPARE(event.getProperty("uid").toByteArray(), QByteArray("testuid")); event.setProperty("summary", "summaryValue"); auto notifier = QSharedPointer::create("sink.dummy.instance1", "sink.dummy"); bool gotNotification = false; int duration = 0; notifier->registerHandler([&gotNotification, &duration, &time](const Sink::Notification ¬ification) { if (notification.type == Sink::Notification::RevisionUpdate) { gotNotification = true; duration = time.elapsed(); } }); Sink::Store::create(event).exec(); // Wait for notification QUICK_TRY_VERIFY(gotNotification); HAWD::Dataset dataset("dummy_responsiveness", m_hawdState); HAWD::Dataset::Row row = dataset.row(); row.setValue("responsetime", duration); dataset.insertRow(row); HAWD::Formatter::print(dataset); VERIFYEXEC(Sink::ResourceControl::shutdown("sink.dummy.instance1")); } void testWriteToFacade() { VERIFYEXEC(Sink::Store::removeDataFromDisk("sink.dummy.instance1")); QTime time; time.start(); QList> waitCondition; for (int i = 0; i < num; i++) { Sink::ApplicationDomain::Event event("sink.dummy.instance1"); event.setProperty("uid", "testuid"); QCOMPARE(event.getProperty("uid").toByteArray(), QByteArray("testuid")); event.setProperty("summary", "summaryValue"); waitCondition << Sink::Store::create(event).exec(); } KAsync::waitForCompletion(waitCondition).exec().waitForFinished(); auto appendTime = time.elapsed(); // Ensure everything is processed { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); } auto allProcessedTime = time.elapsed(); HAWD::Dataset dataset("dummy_write_to_facade", m_hawdState); HAWD::Dataset::Row row = dataset.row(); row.setValue("rows", num); row.setValue("append", (qreal)num / appendTime); row.setValue("total", (qreal)num / allProcessedTime); dataset.insertRow(row); HAWD::Formatter::print(dataset); } void testQueryByUid() { QTime time; time.start(); // Measure query { time.start(); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter("uid", Sink::Query::Comparator("testuid")); auto model = Sink::Store::loadModel(query); QUICK_TRY_VERIFY(model->rowCount(QModelIndex()) == num); } auto queryTime = time.elapsed(); HAWD::Dataset dataset("dummy_query_by_uid", m_hawdState); HAWD::Dataset::Row row = dataset.row(); row.setValue("rows", num); row.setValue("read", (qreal)num / queryTime); dataset.insertRow(row); HAWD::Formatter::print(dataset); } // This allows to run individual parts without doing a cleanup, but still cleaning up normally void testCleanupForCompleteTest() { VERIFYEXEC(Sink::Store::removeDataFromDisk("sink.dummy.instance1")); } private: HAWD::State m_hawdState; }; QTEST_MAIN(DummyResourceBenchmark) #include "dummyresourcebenchmark.moc" diff --git a/tests/dummyresourcewritebenchmark.cpp b/tests/dummyresourcewritebenchmark.cpp index e0ec5036..1b44d354 100644 --- a/tests/dummyresourcewritebenchmark.cpp +++ b/tests/dummyresourcewritebenchmark.cpp @@ -1,299 +1,298 @@ #include #include #include #include #include "dummyresource/resourcefactory.h" -#include "dummyresource/domainadaptor.h" #include "store.h" #include "commands.h" #include "entitybuffer.h" #include "log.h" #include "resourceconfig.h" #include "definitions.h" #include "facadefactory.h" #include "adaptorfactoryregistry.h" #include "hawd/dataset.h" #include "hawd/formatter.h" #include "event_generated.h" #include "mail_generated.h" #include "entity_generated.h" #include "metadata_generated.h" #include "createentity_generated.h" #include "getrssusage.h" #include "utils.h" #include static QByteArray createEntityBuffer(size_t attachmentSize, int &bufferSize) { flatbuffers::FlatBufferBuilder eventFbb; eventFbb.Clear(); { auto msg = KMime::Message::Ptr::create(); msg->subject()->from7BitString("Some subject"); msg->setBody("This is the body now."); msg->assemble(); const auto data = msg->encodedContent(); auto summary = eventFbb.CreateString("summary"); auto mimeMessage = eventFbb.CreateString(data.constData(), data.length()); Sink::ApplicationDomain::Buffer::MailBuilder eventBuilder(eventFbb); eventBuilder.add_subject(summary); eventBuilder.add_messageId(summary); eventBuilder.add_mimeMessage(mimeMessage); Sink::ApplicationDomain::Buffer::FinishMailBuffer(eventFbb, eventBuilder.Finish()); } flatbuffers::FlatBufferBuilder entityFbb; Sink::EntityBuffer::assembleEntityBuffer(entityFbb, 0, 0, 0, 0, eventFbb.GetBufferPointer(), eventFbb.GetSize()); bufferSize = entityFbb.GetSize(); flatbuffers::FlatBufferBuilder fbb; auto type = fbb.CreateString(Sink::ApplicationDomain::getTypeName().toStdString().data()); auto delta = fbb.CreateVector(entityFbb.GetBufferPointer(), entityFbb.GetSize()); Sink::Commands::CreateEntityBuilder builder(fbb); builder.add_domainType(type); builder.add_delta(delta); auto location = builder.Finish(); Sink::Commands::FinishCreateEntityBuffer(fbb, location); return QByteArray(reinterpret_cast(fbb.GetBufferPointer()), fbb.GetSize()); } /** * Benchmark writing in the synchronizer process. */ class DummyResourceWriteBenchmark : public QObject { Q_OBJECT QList mRssGrowthPerEntity; QList mTimePerEntity; QDateTime mTimeStamp{QDateTime::currentDateTimeUtc()}; void writeInProcess(int num, const QDateTime ×tamp) { DummyResource::removeFromDisk("sink.dummy.instance1"); QTime time; time.start(); DummyResource resource(Sink::ResourceContext{"sink.dummy.instance1", "sink.dummy", Sink::AdaptorFactoryRegistry::instance().getFactories("sink.dummy")}); int bufferSize = 0; auto command = createEntityBuffer(0, bufferSize); const auto startingRss = getCurrentRSS(); for (int i = 0; i < num; i++) { resource.processCommand(Sink::Commands::CreateEntityCommand, command); } auto appendTime = time.elapsed(); Q_UNUSED(appendTime); auto bufferSizeTotal = bufferSize * num; // Wait until all messages have been processed resource.processAllMessages().exec().waitForFinished(); auto allProcessedTime = time.elapsed(); const auto finalRss = getCurrentRSS(); const auto rssGrowth = finalRss - startingRss; // Since the database is memory mapped it is attributted to the resident set size. const auto rssWithoutDb = finalRss - DummyResource::diskUsage("sink.dummy.instance1"); const auto peakRss = getPeakRSS(); // How much peak deviates from final rss in percent const auto percentageRssError = static_cast(peakRss - finalRss) * 100.0 / static_cast(finalRss); auto rssGrowthPerEntity = rssGrowth / num; std::cout << "Current Rss usage [kb]: " << finalRss / 1024 << std::endl; std::cout << "Peak Rss usage [kb]: " << peakRss / 1024 << std::endl; std::cout << "Rss growth [kb]: " << rssGrowth / 1024 << std::endl; std::cout << "Rss growth per entity [byte]: " << rssGrowthPerEntity << std::endl; std::cout << "Rss without db [kb]: " << rssWithoutDb / 1024 << std::endl; std::cout << "Percentage peak rss error: " << percentageRssError << std::endl; auto onDisk = Sink::Storage::DataStore(Sink::storageLocation(), "sink.dummy.instance1", Sink::Storage::DataStore::ReadOnly).diskUsage(); auto writeAmplification = static_cast(onDisk) / static_cast(bufferSizeTotal); std::cout << "On disk [kb]: " << onDisk / 1024 << std::endl; std::cout << "Buffer size total [kb]: " << bufferSizeTotal / 1024 << std::endl; std::cout << "Write amplification: " << writeAmplification << std::endl; mTimePerEntity << static_cast(allProcessedTime) / static_cast(num); mRssGrowthPerEntity << rssGrowthPerEntity; { HAWD::Dataset dataset("dummy_write_perf", m_hawdState); HAWD::Dataset::Row row = dataset.row(); row.setValue("rows", num); row.setValue("append", (qreal)num/appendTime); row.setValue("total", (qreal)num/allProcessedTime); row.setTimestamp(timestamp); dataset.insertRow(row); HAWD::Formatter::print(dataset); } { HAWD::Dataset dataset("dummy_write_memory", m_hawdState); HAWD::Dataset::Row row = dataset.row(); row.setValue("rows", num); row.setValue("rss", QVariant::fromValue(finalRss / 1024)); row.setValue("peakRss", QVariant::fromValue(peakRss / 1024)); row.setValue("percentagePeakRssError", percentageRssError); row.setValue("rssGrowthPerEntity", QVariant::fromValue(rssGrowthPerEntity)); row.setValue("rssWithoutDb", rssWithoutDb / 1024); row.setTimestamp(timestamp); dataset.insertRow(row); HAWD::Formatter::print(dataset); } { HAWD::Dataset dataset("dummy_write_disk", m_hawdState); HAWD::Dataset::Row row = dataset.row(); row.setValue("rows", num); row.setValue("onDisk", onDisk / 1024); row.setValue("bufferSize", bufferSizeTotal / 1024); row.setValue("writeAmplification", writeAmplification); row.setTimestamp(timestamp); dataset.insertRow(row); HAWD::Formatter::print(dataset); } // Print memory layout, RSS is what is in memory // std::system("exec pmap -x \"$PPID\""); } void testDiskUsage(int num) { auto resourceId = "testDiskUsage"; DummyResource::removeFromDisk(resourceId); { DummyResource resource(Sink::ResourceContext{resourceId, "sink.dummy", Sink::AdaptorFactoryRegistry::instance().getFactories("sink.dummy")}); int bufferSize = 0; auto command = createEntityBuffer(1000, bufferSize); for (int i = 0; i < num; i++) { resource.processCommand(Sink::Commands::CreateEntityCommand, command); } // Wait until all messages have been processed resource.processAllMessages().exec().waitForFinished(); } qint64 totalDbSizes = 0; qint64 totalKeysAndValues = 0; QMap dbSizes; Sink::Storage::DataStore storage(Sink::storageLocation(), resourceId, Sink::Storage::DataStore::ReadOnly); auto transaction = storage.createTransaction(Sink::Storage::DataStore::ReadOnly); auto stat = transaction.stat(); std::cout << "Free pages: " << stat.freePages << std::endl; std::cout << "Total pages: " << stat.totalPages << std::endl; auto totalUsedSize = stat.pageSize * (stat.totalPages - stat.freePages); std::cout << "Used size: " << totalUsedSize << std::endl; auto freeDbSize = stat.pageSize * (stat.freeDbStat.leafPages + stat.freeDbStat.overflowPages + stat.freeDbStat.branchPages); std::cout << "Free db size: " << freeDbSize << std::endl; auto mainDbSize = stat.pageSize * (stat.mainDbStat.leafPages + stat.mainDbStat.overflowPages + stat.mainDbStat.branchPages); std::cout << "Main db size: " << mainDbSize << std::endl; totalDbSizes += mainDbSize; QList databases = transaction.getDatabaseNames(); for (const auto &databaseName : databases) { auto db = transaction.openDatabase(databaseName); const auto size = db.getSize(); dbSizes.insert(databaseName, size); totalDbSizes += size; qint64 keySizes = 0; qint64 valueSizes = 0; db.scan({}, [&] (const QByteArray &key, const QByteArray &data) { keySizes += key.size(); valueSizes += data.size(); return true; }, [&](const Sink::Storage::DataStore::Error &e) { qWarning() << "Error while reading" << e; }, false, false); auto s = db.stat(); auto usedPages = (s.leafPages + s.branchPages + s.overflowPages); std::cout << std::endl; std::cout << "Db: " << databaseName.toStdString() << (db.allowsDuplicates() ? " DUP" : "") << std::endl; std::cout << "Used pages " << usedPages << std::endl; std::cout << "Used size " << (keySizes + valueSizes) / 4096.0 << std::endl; std::cout << "Entries " << s.numEntries << std::endl; totalKeysAndValues += (keySizes + valueSizes); } std::cout << std::endl; auto mainStoreOnDisk = Sink::Storage::DataStore(Sink::storageLocation(), resourceId, Sink::Storage::DataStore::ReadOnly).diskUsage(); auto totalOnDisk = DummyResource::diskUsage(resourceId); std::cout << "Calculated key + value size: " << totalKeysAndValues << std::endl; std::cout << "Calculated total db sizes: " << totalDbSizes << std::endl; std::cout << "Main store on disk: " << mainStoreOnDisk << std::endl; std::cout << "Total on disk: " << totalOnDisk << std::endl; std::cout << "Used size amplification: " << static_cast(totalUsedSize) / static_cast(totalKeysAndValues) << std::endl; std::cout << "Write amplification: " << static_cast(mainStoreOnDisk) / static_cast(totalKeysAndValues) << std::endl; std::cout << std::endl; } private slots: void initTestCase() { Sink::Log::setDebugOutputLevel(Sink::Log::Warning); auto factory = Sink::ResourceFactory::load("sink.dummy"); QVERIFY(factory); } void cleanup() { } void runBenchmarks() { writeInProcess(5000, mTimeStamp); } void ensureUsedMemoryRemainsStable() { auto rssStandardDeviation = sqrt(variance(mRssGrowthPerEntity)); auto timeStandardDeviation = sqrt(variance(mTimePerEntity)); HAWD::Dataset dataset("dummy_write_summary", m_hawdState); HAWD::Dataset::Row row = dataset.row(); row.setValue("rssStandardDeviation", rssStandardDeviation); row.setValue("rssMaxDifference", maxDifference(mRssGrowthPerEntity)); row.setValue("timeStandardDeviation", timeStandardDeviation); row.setValue("timeMaxDifference", maxDifference(mTimePerEntity)); row.setTimestamp(mTimeStamp); dataset.insertRow(row); HAWD::Formatter::print(dataset); } void testDiskUsage() { testDiskUsage(1000); } // This allows to run individual parts without doing a cleanup, but still cleaning up normally void testCleanupForCompleteTest() { DummyResource::removeFromDisk("sink.dummy.instance1"); } private: HAWD::State m_hawdState; }; QTEST_MAIN(DummyResourceWriteBenchmark) #include "dummyresourcewritebenchmark.moc"