Differential D12746 Diff 45462 kdevplatform/language/duchain/topducontextdynamicdata_p.cpp

Changeset View

Standalone View

kdevplatform/language/duchain/topducontextdynamicdata_p.cpp

This file was added.

		1		/* This is part of KDevelop
		2
		3		Copyright 2018 R.J.V. Bertin <rjvbertin@gmail.com>
		4
		5		This library is free software; you can redistribute it and/or
		6		modify it under the terms of the GNU Library General Public
		7		License version 2 as published by the Free Software Foundation.
		8
		9		This library is distributed in the hope that it will be useful,
		10		but WITHOUT ANY WARRANTY; without even the implied warranty of
		11		MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
		12		Library General Public License for more details.
		13
		14		You should have received a copy of the GNU Library General Public License
		15		along with this library; see the file COPYING.LIB. If not, write to
		16		the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
		17		Boston, MA 02110-1301, USA.
		18		*/
		19
		20		#include "topducontextdynamicdata_p.h"
		21		#include "topducontextdynamicdata.h"
		22
		23		#include <QFileInfo>
		24
		25		#ifndef KDEV_TOPCONTEXTS_USE_FILES
		26		#include <lmdb++.h>
		27		// we roll our own compression
		28		#include <lz4.h>
		29		#endif
		30
		31		#include <debug.h>
		32
		33		//#define DEBUG_DATA_INFO
		34
		35		using namespace KDevelop;
		36
		37		#ifndef KDEV_TOPCONTEXTS_USE_FILES
		38
		39		bool TopDUContextDB::open(QIODevice::OpenMode mode, const QString &backendName)
		40		{
		41		if (isValid() && !m_currentKey.isEmpty()) {
		42		int lmMode = mode == QIODevice::ReadOnly ? MDB_RDONLY : 0;
		43		if (lmMode == MDB_RDONLY && !currentKeyExists()) {
		44		// migration: see if the index file exists
		45		if (migrateFromFile()) {
		46		return true;
		47		}
		48		m_errorString = QStringLiteral("No item #") + QByteArray::number(m_currentIndex) + QStringLiteral(" in database");
		49		return false;
		50		}
		51		m_mode = lmMode;
		52		m_currentValue.clear();
		53		m_currentLen = 0;
		54		m_readCursor = -1;
		55		m_errorString.clear();
		56		return true;
		57		}
		58		m_errorString = QStringLiteral("%1 database backend not initialised properly").arg(backendName);
		59		return false;
		60		}
		61
		62		QString TopDUContextDB::fileName() const
		63		{
		64		return TopDUContextDynamicData::basePath() + "...#" + QByteArray::number(m_currentIndex);
		65		}
		66
		67		QString TopDUContextDB::errorString() const
		68		{
		69		return m_errorString;
		70		}
		71
		72		bool TopDUContextDB::resize(qint64)
		73		{
		74		return m_mode != MDB_RDONLY;
		75		}
		76
		77		qint64 TopDUContextDB::write(const char *data, qint64 len)
		78		{
		79		if (m_mode != MDB_RDONLY) {
		80		m_currentValue.append(QByteArray::fromRawData(data, len));
		81		m_currentLen += len;
		82		return len;
		83		}
		84		return 0;
		85		}
		86
		87		// read the current key value into m_currentValue if necessary, and return the
		88		// requested @p maxSize bytes from the current read position in @p data. Update
		89		// the read position afterwards, and reset m_currentValue when all data has
		90		// been returned.
		91		// Special case: data==NULL and maxSize==-1; return the number of remaining bytes
		92		// and do NOT reset m_currentValue
		93		qint64 TopDUContextDB::read(char *data, qint64 maxSize)
		94		{
		95		if (isValid() && !m_currentKey.isEmpty() && m_mode == MDB_RDONLY) {
		96		if (!getCurrentKeyValue()) {
		97		return -1;
		98		}
		99		if (m_readCursor >= 0 && m_readCursor < m_currentLen) {
		100		qint64 rlen = m_currentLen - m_readCursor;
		101		if (Q_LIKELY(maxSize >= 0)) {
		102		if (maxSize < rlen) {
		103		rlen = maxSize;
		104		}
		105		const char *val = m_currentValue.constData();
		106		memcpy(data, &val[m_readCursor], rlen);
		107		m_readCursor += rlen;
		108		if (m_readCursor >= m_currentLen) {
		109		// all read, clear the cache
		110		m_currentValue.clear();
		111		m_currentLen = 0;
		112		m_readCursor = -1;
		113		}
		114		} else {
		115		// special case: don't update m_readCursor;
		116		}
		117		return rlen;
		118		}
		119		}
		120		return -1;
		121		}
		122
		123		QByteArray TopDUContextDB::read(qint64 maxSize)
		124		{
		125		QByteArray data;
		126		data.resize(maxSize);
		127		auto len = read(data.data(), maxSize);
		128		data.resize(len >= 0 ? len : 0);
		129		return data;
		130		}
		131
		132		// Reads all the remaining data, returned as a QByteArray
		133		QByteArray TopDUContextDB::readAll()
		134		{
		135		QByteArray data;
		136		auto readLen = read(nullptr, -1);
		137		if (readLen > 0) {
		138		// should always be true:
		139		if (Q_LIKELY(m_readCursor >= 0 && m_readCursor + readLen <= m_currentValue.size())) {
		140		if (m_readCursor == 0) {
		141		data = m_currentValue;
		142		} else {
		143		data.resize(readLen);
		144		const char *val = m_currentValue.constData();
		145		memcpy(data.data(), &val[m_readCursor], readLen);
		146		}
		147		} else {
		148		qCWarning(LANGUAGE) << Q_FUNC_INFO << "m_readCursor=" << m_readCursor << "readLen=" << readLen
		149		<< "m_currentValue.size=" << m_currentValue.size();
		150		if (readLen == m_currentValue.size()) {
		151		// m_readCursor should have been 0!!!
		152		qCWarning(LANGUAGE) << "\tm_readCursor should have been 0!!";
		153		data = m_currentValue;
		154		}
		155		}
		156		// all read, clear the cache
		157		m_currentValue.clear();
		158		m_currentLen = 0;
		159		m_readCursor = -1;
		160		}
		161		return data;
		162		}
		163
		164		qint64 TopDUContextDB::pos() const
		165		{
		166		return m_readCursor < 0 ? 0 : m_readCursor;
		167		}
		168
		169		bool TopDUContextDB::seek(qint64 pos)
		170		{
		171		if (pos <= m_currentLen) {
		172		m_readCursor = pos;
		173		return true;
		174		}
		175		return false;
		176		}
		177
		178		qint64 TopDUContextDB::size()
		179		{
		180		if (!m_currentLen && m_mode == MDB_RDONLY) {
		181		// cache the key value
		182		read(nullptr, -1);
		183		}
		184		return m_currentLen;
		185		}
		186
		187		QByteArray TopDUContextDB::indexKey(uint idx)
		188		{
		189		return QByteArray::number(idx);
		190		}
		191
		192		// NB: @p idx must point to a variable that will not be outlived by the returned QByteArray!
		193		QByteArray TopDUContextDB::indexKey(uint *idx)
		194		{
		195		return QByteArray::fromRawData(reinterpret_cast<const char *>(idx), sizeof(uint));
		196		}
		197
		198		bool TopDUContextDB::migrateFromFile()
		199		{
		200		TopDUContextFile migrateFile(m_currentIndex);
		201		if (migrateFile.open(QIODevice::ReadOnly)) {
		202		// should we care about empty files here?
		203		qCDebug(LANGUAGE) << "Migrating" << migrateFile.fileName();
		204		const QByteArray content = migrateFile.readAll();
		205		migrateFile.close();
		206		m_mode = 0;
		207		m_currentValue = content;
		208		m_currentLen = content.size();
		209		// commit() will reset the key so we need to cache it
		210		const QByteArray key = m_currentKey;
		211		m_errorString.clear();
		212		commit();
		213		if (m_errorString.isEmpty()) {
		214		// migration was successful, remove the file
		215		QFile::remove(migrateFile.fileName());
		216		}
		217		m_errorString.clear();
		218		// take care that we don't have to read the data back in
		219		m_currentKey = key;
		220		m_currentValue = content;
		221		m_currentLen = content.size();
		222		m_readCursor = 0;
		223		m_mode = MDB_RDONLY;
		224		return true;
		225		}
		226		return false;
		227		}
		228
		229		// TopDUContextLMDB : wraps the QFile API needed for TopDUContexts around LMDB
		230
		231		static QString lmdbxx_exception_handler(const lmdb::error &e, const QString &operation)
		232		{
		233		const QString msg = QStringLiteral("LMDB exception in \"%1\": %2").arg(operation).arg(e.what());
		234		qCWarning(LANGUAGE) << msg;
		235		if (qEnvironmentVariableIsSet("KDEV_TOPCONTEXTS_STORE_FAILURE_ABORT")) {
		236		qFatal(msg.toLatin1().constData());
		237		}
		238		return msg;
		239		}
		240
		241		// there is exactly 1 topcontexts directory per session, so we can make do with a single
		242		// global static LMDB env instance which is not exported at all (= no need to include
		243		// lmdb.h and/or lmdbxx.h in our own headerfile).
		244
		245		static double compRatioSum = 0;
		246		static size_t compRatioN = 0;
		247		static size_t uncompN = 0;
		248
		249		static void printCompRatio()
		250		{
		251		if (compRatioN) {
		252		fprintf(stderr, "average LZ4 compression ratio: %g; %lu compressed of %lu\n",
		253		compRatioSum / compRatioN, compRatioN, compRatioN + uncompN);
		254		}
		255		}
		256
		257		class LMDBHook
		258		{
		259		public:
		260		~LMDBHook()
		261		{
		262		if (s_envExists) {
		263		s_lmdbEnv.close();
		264		s_envExists = false;
		265		delete s_lz4State;
		266		printCompRatio();
		267		}
		268		}
		269
		270		static bool init()
		271		{
		272		if (!s_envExists) {
		273		s_errorString.clear();
		274		try {
		275		s_lmdbEnv = lmdb::env::create();
		276		// Open the environment in async mode. From the documentation:
		277		// if the filesystem preserves write order [...], transactions exhibit ACI
		278		// (atomicity, consistency, isolation) properties and only lose D (durability).
		279		// I.e. database integrity is maintained, but a system crash may undo the final transactions.
		280		// This should be acceptable for caching self-generated data, given how much faster
		281		// transactions become.
		282		s_lmdbEnv.open(TopDUContextDynamicData::basePath().toLatin1().constData(), MDB_NOSYNC);
		283		MDB_envinfo stat;
		284		lmdb::env_info(s_lmdbEnv.handle(), &stat);
		285		if (stat.me_mapsize > s_mapSize) {
		286		s_mapSize = stat.me_mapsize;
		287		}
		288		s_lmdbEnv.set_mapsize(s_mapSize);
		289		s_lz4State = new char[LZ4_sizeofState()];
		290		s_envExists = true;
		291		qCDebug(LANGUAGE) << "s_lmdbEnv=" << s_lmdbEnv << "mapsize=" << stat.me_mapsize << "LZ4 state buffer:" << LZ4_sizeofState();
		292		} catch (const lmdb::error &e) {
		293		s_errorString = lmdbxx_exception_handler(e, QStringLiteral("database creation"));
		294		// as per the documentation: the environment must be closed even if creation failed!
		295		s_lmdbEnv.close();
		296		}
		297		}
		298		return false;
		299		}
		300
		301		inline lmdb::env* instance()
		302		{
		303		return s_envExists? &s_lmdbEnv : nullptr;
		304		}
		305		inline MDB_env* handle()
		306		{
		307		return s_envExists? s_lmdbEnv.handle() : nullptr;
		308		}
		309
		310		void growMapSize()
		311		{
		312		s_mapSize *= 2;
		313		qCDebug(LANGUAGE) << "\tgrowing mapsize to" << s_mapSize;
		314		s_lmdbEnv.set_mapsize(s_mapSize);
		315		}
		316
		317
		318		static lmdb::env s_lmdbEnv;
		319		static bool s_envExists;
		320		static char* s_lz4State;
		321		static size_t s_mapSize;
		322		static QString s_errorString;
		323		};
		324		static LMDBHook LMDB;
		325
		326		lmdb::env LMDBHook::s_lmdbEnv{nullptr};
		327		bool LMDBHook::s_envExists = false;
		328		char *LMDBHook::s_lz4State = nullptr;
		329		// set the initial map size to 64Mb
		330		size_t LMDBHook::s_mapSize = 1024UL * 1024UL * 64UL;
		331		QString LMDBHook::s_errorString;
		332
		333		uint TopDUContextLMDB::s_DbRefCount = 0;
		334
		335		TopDUContextLMDB::TopDUContextLMDB(uint topContextIndex)
		336		{
		337		m_currentIndex = topContextIndex;
		338		if (!LMDB.instance() && Q_LIKELY(QFileInfo(TopDUContextDynamicData::basePath()).isWritable())) {
		339		if (!LMDB.init()) {
		340		m_errorString = LMDB.s_errorString;
		341		}
		342		}
		343		if (LMDB.instance()) {
		344		m_currentKey = indexKey(m_currentIndex);
		345		s_DbRefCount += 1;
		346		}
		347		m_currentLen = -1;
		348		// the remaining member vars are initialised elsewhere intentionally.
		349		}
		350
		351		TopDUContextLMDB::~TopDUContextLMDB()
		352		{
		353		if (LMDB.instance()) {
		354		s_DbRefCount -= 1;
		355		if (s_DbRefCount <= 0) {
		356		s_DbRefCount = 0;
		357		#ifdef DEBUG
		358		flush();
		359		#endif
		360		}
		361		} else {
		362		s_DbRefCount = 0;
		363		}
		364		}
		365
		366		bool TopDUContextLMDB::open(QIODevice::OpenMode mode)
		367		{
		368		return TopDUContextDB::open(mode, QStringLiteral("LMDB"));
		369		}
		370
		371		void TopDUContextLMDB::commit()
		372		{
		373		if (LMDB.instance() && m_mode != MDB_RDONLY) {
		374		if (m_currentValue.size() != m_currentLen) {
		375		// m_currentLen is the true size
		376		qCDebug(LANGUAGE) << "TopDUContextLMDB index" << QByteArray::number(m_currentIndex) << "internal size mismatch:"
		377		<< m_currentValue.size() << "vs" << m_currentLen;
		378		}
		379		char *data;
		380		size_t dataLen = 0;
		381		int lz4BufLen = m_currentLen > 2 * sizeof(qint64) ? LZ4_compressBound(m_currentLen) : 0;
		382		lmdb::val value;
		383		if (lz4BufLen) {
		384		data = new char[lz4BufLen + sizeof(qint64)];
		385		// compress to an qint64-sized offset into the dest buffer; the original size will be stored in
		386		// those first 64 bits.
		387		dataLen = LZ4_compress_fast_extState(LMDB.s_lz4State, m_currentValue.constData(), &data[sizeof(qint64)],
		388		m_currentLen, lz4BufLen, 1);
		389		if (dataLen && dataLen + sizeof(qint64) < m_currentLen) {
		390		LZ4Frame frame;
		391		frame.bytes = data;
		392		frame.qint32Ptr[0] = m_currentLen;
		393		frame.qint32Ptr[1] = dataLen;
		394		value = lmdb::val(data, dataLen + sizeof(qint64));
		395		compRatioSum += double(m_currentLen) / value.size();
		396		compRatioN += 1;
		397		} else {
		398		qCDebug(LANGUAGE) << "Index" << m_currentIndex << "compression failed or useless: m_currentLen=" << m_currentLen
		399		<< "compressedLen=" << dataLen << "LZ4_compressBound=" << lz4BufLen;
		400		delete data;
		401		dataLen = 0;
		402		lz4BufLen = 0;
		403		}
		404		}
		405		if (!dataLen) {
		406		value = lmdb::val(m_currentValue.constData(), m_currentLen);
		407		uncompN += 1;
		408		}
		409		lmdb::val key(m_currentKey.constData(), m_currentKey.size());
		410		try {
		411		auto txn = lmdb::txn::begin(LMDB.handle());
		412		auto dbi = lmdb::dbi::open(txn, nullptr);
		413		try {
		414		lmdb::dbi_put(txn, dbi, key, value);
		415		txn.commit();
		416		} catch (const lmdb::error &e) {
		417		if (e.code() == MDB_MAP_FULL) {
		418		try {
		419		qCDebug(LANGUAGE) << "aborting LMDB write to grow mapsize";
		420		txn.abort();
		421		lmdb::dbi_close(LMDB.handle(), dbi);
		422		LMDB.growMapSize();
		423		commit();
		424		} catch (const lmdb::error &e) {
		425		m_errorString = lmdbxx_exception_handler(e, QStringLiteral("growing mapsize to ")
		426		+ QString::number(LMDB.s_mapSize));
		427		}
		428		} else {
		429		m_errorString = lmdbxx_exception_handler(e, QStringLiteral("committing index ")
		430		+ QByteArray::number(m_currentIndex) + " size " + QString::number(m_currentLen));
		431		}
		432		}
		433		} catch (const lmdb::error &e) {
		434		m_errorString = lmdbxx_exception_handler(e, QStringLiteral("committing index ")
		435		+ QByteArray::number(m_currentIndex) + " size " + QString::number(m_currentLen));
		436		}
		437		m_currentKey.clear();
		438		m_currentValue.clear();
		439		m_currentLen = 0;
		440		if (lz4BufLen && data) {
		441		delete data;
		442		}
		443		}
		444		}
		445
		446		bool TopDUContextLMDB::flush()
		447		{
		448		if (LMDB.instance() && m_mode != MDB_RDONLY) {
		449		try {
		450		return mdb_env_sync(LMDB.handle(), true) == MDB_SUCCESS;
		451		} catch (const lmdb::error &e) {
		452		m_errorString = lmdbxx_exception_handler(e, QStringLiteral("database flush"));
		453		}
		454		}
		455		return false;
		456		}
		457
		458		bool TopDUContextLMDB::getCurrentKeyValue()
		459		{
		460		// we only return false if a read error occurred; if a key doesn't exist
		461		// m_currentValue will remain empty.
		462		bool ret = true;
		463		if (m_currentValue.isEmpty()) {
		464		// read the key value from storage into cache
		465		try {
		466		auto rtxn = lmdb::txn::begin(LMDB.handle(), nullptr, MDB_RDONLY);
		467		auto dbi = lmdb::dbi::open(rtxn, nullptr);
		468		auto cursor = lmdb::cursor::open(rtxn, dbi);
		469		lmdb::val key(m_currentKey.constData(), m_currentKey.size());
		470		lmdb::val val {};
		471		bool found = cursor.get(key, val, MDB_SET);
		472		if (found) {
		473		bool isRaw = true;
		474		if (val.size() > sizeof(qint64)) {
		475		LZ4Frame frame;
		476		frame.bytes = val.data();
		477		int orgSize = frame.qint32Ptr[0];
		478		int compressedSize = val.size() - sizeof(qint64);
		479		if (orgSize > 0 && frame.qint32Ptr[1] == compressedSize) {
		480		// size m_currentValue so decompression can go into the final destination directly
		481		m_currentValue.resize(orgSize);
		482		if (LZ4_decompress_safe(&frame.bytes[sizeof(qint64)], m_currentValue.data(), compressedSize, orgSize) == orgSize) {
		483		isRaw = false;
		484		m_currentLen = m_currentValue.size();
		485		} else {
		486		// qCDebug(LANGUAGE) << "Index" << m_currentIndex << "failed LZ4 decompression from size" << compressedSize << "to size" << orgSize;
		487		}
		488		}
		489		}
		490		if (isRaw) {
		491		// qCDebug(LANGUAGE) << "Index" << m_currentIndex << "is uncompressed, size=" << val.size();
		492		m_currentValue = QByteArray(val.data(), val.size());
		493		m_currentLen = m_currentValue.size();
		494		}
		495		m_readCursor = 0;
		496		}
		497		cursor.close();
		498		rtxn.abort();
		499		} catch (const lmdb::error &e) {
		500		m_errorString = lmdbxx_exception_handler(e, QStringLiteral("reading index ") + QByteArray::number(m_currentIndex));
		501		ret = false;
		502		}
		503		}
		504		return ret;
		505		}
		506
		507		bool TopDUContextLMDB::isValid()
		508		{
		509		return LMDB.instance();
		510		}
		511
		512		bool TopDUContextLMDB::currentKeyExists()
		513		{
		514		return exists(m_currentKey);
		515		}
		516
		517		bool TopDUContextLMDB::exists(const QByteArray &key)
		518		{
		519		if (LMDB.instance()) {
		520		try {
		521		auto rtxn = lmdb::txn::begin(LMDB.handle(), nullptr, MDB_RDONLY);
		522		auto dbi = lmdb::dbi::open(rtxn, nullptr);
		523		auto cursor = lmdb::cursor::open(rtxn, dbi);
		524		lmdb::val k(key.constData(), key.size());
		525		bool ret = cursor.get(k, nullptr, MDB_SET);
		526		cursor.close();
		527		rtxn.abort();
		528		return ret;
		529		} catch (const lmdb::error &e) {
		530		lmdbxx_exception_handler(e, QStringLiteral("checking for index") + key);
		531		}
		532		}
		533		return false;
		534		}
		535
		536		QString TopDUContextLMDB::fileName() const
		537		{
		538		return TopDUContextDynamicData::basePath() + "data.mdb" + ":#" + QByteArray::number(m_currentIndex);
		539		}
		540
		541		bool TopDUContextLMDB::exists(uint topContextIndex)
		542		{
		543		return exists(indexKey(topContextIndex));
		544		}
		545
		546		bool TopDUContextLMDB::remove(uint topContextIndex)
		547		{
		548		if (LMDB.instance()) {
		549		const auto key = indexKey(topContextIndex);
		550		lmdb::val k {key.constData(), static_cast<size_t>(key.size())};
		551		try {
		552		auto txn = lmdb::txn::begin(LMDB.handle());
		553		auto dbi = lmdb::dbi::open(txn, nullptr);
		554		bool ret = lmdb::dbi_del(txn, dbi, k, nullptr);
		555		txn.commit();
		556		// also remove the file if it (still) exists
		557		QFile::remove(TopDUContextDynamicData::pathForTopContext(topContextIndex));
		558		return ret;
		559		} catch (const lmdb::error &e) {
		560		lmdbxx_exception_handler(e, QStringLiteral("removing index %1").arg(topContextIndex));
		561		}
		562		}
		563		return false;
		564		}
		565
		566		#endif // !KDEV_TOPCONTEXTS_USE_FILES
		567
		568		// TopDUContextFile : thin wrapper around the QFile API needed for TopDUContexts
		569		// so TopDUContextLMDB can be used as a drop-in replacement instead of this class.
		570		TopDUContextFile::TopDUContextFile(uint topContextIndex)
		571		: QFile(TopDUContextDynamicData::pathForTopContext(topContextIndex))
		572		{
		573		}
		574
		575		bool TopDUContextFile::exists(uint topContextIndex)
		576		{
		577		return QFile::exists(TopDUContextDynamicData::pathForTopContext(topContextIndex));
		578		}
		579
		580		bool TopDUContextFile::remove(uint topContextIndex)
		581		{
		582		return QFile::remove(TopDUContextDynamicData::pathForTopContext(topContextIndex));
		583		}
		584
		585		void TopDUContextFile::commit()
		586		{
		587		QFile::close();
		588		}