Differential D12746 Diff 38266 kdevplatform/language/duchain/topducontextdynamicdata_p.cpp

Changeset View

Standalone View

kdevplatform/language/duchain/topducontextdynamicdata_p.cpp

This file was added.

		1		/* This is part of KDevelop
		2
		3		Copyright 2018 R.J.V. Bertin <rjvbertin@gmail.com>
		4
		5		This library is free software; you can redistribute it and/or
		6		modify it under the terms of the GNU Library General Public
		7		License version 2 as published by the Free Software Foundation.
		8
		9		This library is distributed in the hope that it will be useful,
		10		but WITHOUT ANY WARRANTY; without even the implied warranty of
		11		MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
		12		Library General Public License for more details.
		13
		14		You should have received a copy of the GNU Library General Public License
		15		along with this library; see the file COPYING.LIB. If not, write to
		16		the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
		17		Boston, MA 02110-1301, USA.
		18		*/
		19
		20		#include "topducontextdynamicdata_p.h"
		21		#include "topducontextdynamicdata.h"
		22
		23		#include <QFileInfo>
		24
		25		#ifndef KDEV_TOPCONTEXTS_USE_FILES
		26		#ifdef KDEV_TOPCONTEXTS_USE_LMDB
		27		#include <lmdb++.h>
		28		// we roll our own compression
		29		#include <lz4.h>
		30		#else
		31		#define MDB_RDONLY 0x20000
		32		#endif
		33		#ifdef KDEV_TOPCONTEXTS_USE_LEVELDB
		34		#include <leveldb/db.h>
		35		#include <leveldb/comparator.h>
		36		#endif
		37		#ifdef KDEV_TOPCONTEXTS_USE_KYOTO
		38		#include <kcpolydb.h>
		39		#endif
		40		#endif
		41
		42		#include <debug.h>
		43
		44		//#define DEBUG_DATA_INFO
		45
		46		using namespace KDevelop;
		47
		48		#ifndef KDEV_TOPCONTEXTS_USE_FILES
		49
		50		bool TopDUContextDB::open(QIODevice::OpenMode mode, const QString &backendName)
		51		{
		52		if (isValid() && !m_currentKey.isEmpty()) {
		53		int lmMode = mode == QIODevice::ReadOnly ? MDB_RDONLY : 0;
		54		if (lmMode == MDB_RDONLY && !currentKeyExists()) {
		55		// migration: see if the index file exists
		56		if (migrateFromFile()) {
		57		return true;
		58		}
		59		m_errorString = QStringLiteral("No item #") + QByteArray::number(m_currentIndex) + QStringLiteral(" in database");
		60		return false;
		61		}
		62		m_mode = lmMode;
		63		m_currentValue.clear();
		64		m_currentLen = 0;
		65		m_readCursor = -1;
		66		m_errorString.clear();
		67		return true;
		68		}
		69		m_errorString = QStringLiteral("%1 database backend not initialised properly").arg(backendName);
		70		return false;
		71		}
		72
		73		QString TopDUContextDB::fileName() const
		74		{
		75		return TopDUContextDynamicData::basePath() + "...#" + QByteArray::number(m_currentIndex);
		76		}
		77
		78		QString TopDUContextDB::errorString() const
		79		{
		80		return m_errorString;
		81		}
		82
		83		bool TopDUContextDB::resize(qint64)
		84		{
		85		return m_mode != MDB_RDONLY;
		86		}
		87
		88		qint64 TopDUContextDB::write(const char *data, qint64 len)
		89		{
		90		if (m_mode != MDB_RDONLY) {
		91		m_currentValue.append(QByteArray::fromRawData(data, len));
		92		m_currentLen += len;
		93		return len;
		94		}
		95		return 0;
		96		}
		97
		98		// read the current key value into m_currentValue if necessary, and return the
		99		// requested @p maxSize bytes from the current read position in @p data. Update
		100		// the read position afterwards, and reset m_currentValue when all data has
		101		// been returned.
		102		// Special case: data==NULL and maxSize==-1; return the number of remaining bytes
		103		// and do NOT reset m_currentValue
		104		qint64 TopDUContextDB::read(char *data, qint64 maxSize)
		105		{
		106		if (isValid() && !m_currentKey.isEmpty() && m_mode == MDB_RDONLY) {
		107		if (!getCurrentKeyValue()) {
		108		return -1;
		109		}
		110		if (m_readCursor >= 0 && m_readCursor < m_currentLen) {
		111		qint64 rlen = m_currentLen - m_readCursor;
		112		if (Q_LIKELY(maxSize >= 0)) {
		113		if (maxSize < rlen) {
		114		rlen = maxSize;
		115		}
		116		const char *val = m_currentValue.constData();
		117		memcpy(data, &val[m_readCursor], rlen);
		118		m_readCursor += rlen;
		119		if (m_readCursor >= m_currentLen) {
		120		// all read, clear the cache
		121		m_currentValue.clear();
		122		m_currentLen = 0;
		123		m_readCursor = -1;
		124		}
		125		} else {
		126		// special case: don't update m_readCursor;
		127		}
		128		return rlen;
		129		}
		130		}
		131		return -1;
		132		}
		133
		134		QByteArray TopDUContextDB::read(qint64 maxSize)
		135		{
		136		QByteArray data;
		137		data.resize(maxSize);
		138		auto len = read(data.data(), maxSize);
		139		data.resize(len >= 0 ? len : 0);
		140		return data;
		141		}
		142
		143		// Reads all the remaining data, returned as a QByteArray
		144		QByteArray TopDUContextDB::readAll()
		145		{
		146		QByteArray data;
		147		auto readLen = read(nullptr, -1);
		148		if (readLen > 0) {
		149		// should always be true:
		150		if (Q_LIKELY(m_readCursor >= 0 && m_readCursor + readLen <= m_currentValue.size())) {
		151		if (m_readCursor == 0) {
		152		data = m_currentValue;
		153		} else {
		154		data.resize(readLen);
		155		const char *val = m_currentValue.constData();
		156		memcpy(data.data(), &val[m_readCursor], readLen);
		157		}
		158		} else {
		159		qCWarning(LANGUAGE) << Q_FUNC_INFO << "m_readCursor=" << m_readCursor << "readLen=" << readLen
		160		<< "m_currentValue.size=" << m_currentValue.size();
		161		if (readLen == m_currentValue.size()) {
		162		// m_readCursor should have been 0!!!
		163		qCWarning(LANGUAGE) << "\tm_readCursor should have been 0!!";
		164		data = m_currentValue;
		165		}
		166		}
		167		// all read, clear the cache
		168		m_currentValue.clear();
		169		m_currentLen = 0;
		170		m_readCursor = -1;
		171		}
		172		return data;
		173		}
		174
		175		qint64 TopDUContextDB::pos() const
		176		{
		177		return m_readCursor < 0 ? 0 : m_readCursor;
		178		}
		179
		180		bool TopDUContextDB::seek(qint64 pos)
		181		{
		182		if (pos <= m_currentLen) {
		183		m_readCursor = pos;
		184		return true;
		185		}
		186		return false;
		187		}
		188
		189		qint64 TopDUContextDB::size()
		190		{
		191		if (!m_currentLen && m_mode == MDB_RDONLY) {
		192		// cache the key value
		193		read(nullptr, -1);
		194		}
		195		return m_currentLen;
		196		}
		197
		198		QByteArray TopDUContextDB::indexKey(uint idx)
		199		{
		200		return QByteArray::number(idx);
		201		}
		202
		203		// NB: @p idx must point to a variable that will not be outlived by the returned QByteArray!
		204		QByteArray TopDUContextDB::indexKey(uint *idx)
		205		{
		206		return QByteArray::fromRawData(reinterpret_cast<const char *>(idx), sizeof(uint));
		207		}
		208
		209		bool TopDUContextDB::migrateFromFile()
		210		{
		211		TopDUContextFile migrateFile(m_currentIndex);
		212		if (migrateFile.open(QIODevice::ReadOnly)) {
		213		// should we care about empty files here?
		214		qCDebug(LANGUAGE) << "Migrating" << migrateFile.fileName();
		215		const QByteArray content = migrateFile.readAll();
		216		migrateFile.close();
		217		m_mode = 0;
		218		m_currentValue = content;
		219		m_currentLen = content.size();
		220		// commit() will reset the key so we need to cache it
		221		const QByteArray key = m_currentKey;
		222		m_errorString.clear();
		223		commit();
		224		if (m_errorString.isEmpty()) {
		225		// migration was successful, remove the file
		226		QFile::remove(migrateFile.fileName());
		227		}
		228		m_errorString.clear();
		229		// take care that we don't have to read the data back in
		230		m_currentKey = key;
		231		m_currentValue = content;
		232		m_currentLen = content.size();
		233		m_readCursor = 0;
		234		m_mode = MDB_RDONLY;
		235		return true;
		236		}
		237		return false;
		238		}
		239
		240		#ifdef KDEV_TOPCONTEXTS_USE_LMDB
		241		// TopDUContextLMDB : wraps the QFile API needed for TopDUContexts around LMDB
		242
		243		static QString lmdbxx_exception_handler(const lmdb::error &e, const QString &operation)
		244		{
		245		const QString msg = QStringLiteral("LMDB exception in \"%1\": %2").arg(operation).arg(e.what());
		246		qCWarning(LANGUAGE) << msg;
		247		if (qEnvironmentVariableIsSet("KDEV_TOPCONTEXTS_STORE_FAILURE_ABORT")) {
		248		qFatal(msg.toLatin1());
		249		}
		250		return msg;
		251		}
		252
		253		// there is exactly 1 topcontexts directory per session, so we can make do with a single
		254		// global static LMDB env instance which is not exported at all (= no need to include
		255		// lmdb.h and/or lmdbxx.h in our own headerfile).
		256
		257		static double compRatioSum = 0;
		258		static size_t compRatioN = 0;
		259		static size_t uncompN = 0;
		260
		261		static void printCompRatio()
		262		{
		263		if (compRatioN) {
		264		fprintf(stderr, "average LZ4 compression ratio: %g; %lu compressed of %lu\n",
		265		compRatioSum / compRatioN, compRatioN, compRatioN + uncompN);
		266		}
		267		}
		268
		269		class LMDBHook
		270		{
		271		public:
		272		~LMDBHook()
		273		{
		274		if (s_envExists) {
		275		s_lmdbEnv.close();
		276		s_envExists = false;
		277		delete s_lz4State;
		278		printCompRatio();
		279		}
		280		}
		281
		282		static bool init()
		283		{
		284		if (!s_envExists) {
		285		s_errorString.clear();
		286		try {
		287		s_lmdbEnv = lmdb::env::create();
		288		// Open the environment in async mode. From the documentation:
		289		// if the filesystem preserves write order [...], transactions exhibit ACI
		290		// (atomicity, consistency, isolation) properties and only lose D (durability).
		291		// I.e. database integrity is maintained, but a system crash may undo the final transactions.
		292		// This should be acceptable for caching self-generated data, given how much faster
		293		// transactions become.
		294		s_lmdbEnv.open(TopDUContextDynamicData::basePath().toLatin1().constData(), MDB_NOSYNC);
		295		MDB_envinfo stat;
		296		lmdb::env_info(s_lmdbEnv.handle(), &stat);
		297		if (stat.me_mapsize > s_mapSize) {
		298		s_mapSize = stat.me_mapsize;
		299		}
		300		s_lmdbEnv.set_mapsize(s_mapSize);
		301		s_lz4State = new char[LZ4_sizeofState()];
		302		s_envExists = true;
		303		qCDebug(LANGUAGE) << "s_lmdbEnv=" << s_lmdbEnv << "mapsize=" << stat.me_mapsize << "LZ4 state buffer:" << LZ4_sizeofState();
		304		} catch (const lmdb::error &e) {
		305		s_errorString = lmdbxx_exception_handler(e, QStringLiteral("database creation"));
		306		// as per the documentation: the environment must be closed even if creation failed!
		307		s_lmdbEnv.close();
		308		}
		309		}
		310		return false;
		311		}
		312
		313		inline lmdb::env* instance()
		314		{
		315		return s_envExists? &s_lmdbEnv : nullptr;
		316		}
		317		inline MDB_env* handle()
		318		{
		319		return s_envExists? s_lmdbEnv.handle() : nullptr;
		320		}
		321
		322		void growMapSize()
		323		{
		324		s_mapSize *= 2;
		325		qCDebug(LANGUAGE) << "\tgrowing mapsize to" << s_mapSize;
		326		s_lmdbEnv.set_mapsize(s_mapSize);
		327		}
		328
		329
		330		static lmdb::env s_lmdbEnv;
		331		static bool s_envExists;
		332		static char* s_lz4State;
		333		static size_t s_mapSize;
		334		static QString s_errorString;
		335		};
		336		static LMDBHook LMDB;
		337
		338		lmdb::env LMDBHook::s_lmdbEnv{nullptr};
		339		bool LMDBHook::s_envExists = false;
		340		char *LMDBHook::s_lz4State = nullptr;
		341		// set the initial map size to 64Mb
		342		size_t LMDBHook::s_mapSize = 1024UL * 1024UL * 64UL;
		343		QString LMDBHook::s_errorString;
		344
		345		uint TopDUContextLMDB::s_DbRefCount = 0;
		346
		347		TopDUContextLMDB::TopDUContextLMDB(uint topContextIndex)
		348		{
		349		m_currentIndex = topContextIndex;
		350		if (!LMDB.instance() && Q_LIKELY(QFileInfo(TopDUContextDynamicData::basePath()).isWritable())) {
		351		if (!LMDB.init()) {
		352		m_errorString = LMDB.s_errorString;
		353		}
		354		}
		355		if (LMDB.instance()) {
		356		m_currentKey = indexKey(m_currentIndex);
		357		s_DbRefCount += 1;
		358		}
		359		m_currentLen = -1;
		360		// the remaining member vars are initialised elsewhere intentionally.
		361		}
		362
		363		TopDUContextLMDB::~TopDUContextLMDB()
		364		{
		365		if (LMDB.instance()) {
		366		s_DbRefCount -= 1;
		367		if (s_DbRefCount <= 0) {
		368		s_DbRefCount = 0;
		369		#ifdef DEBUG
		370		flush();
		371		#endif
		372		}
		373		} else {
		374		s_DbRefCount = 0;
		375		}
		376		}
		377
		378		bool TopDUContextLMDB::open(QIODevice::OpenMode mode)
		379		{
		380		return TopDUContextDB::open(mode, QStringLiteral("LMDB"));
		381		}
		382
		383		void TopDUContextLMDB::commit()
		384		{
		385		if (LMDB.instance() && m_mode != MDB_RDONLY) {
		386		if (m_currentValue.size() != m_currentLen) {
		387		// m_currentLen is the true size
		388		qCDebug(LANGUAGE) << "TopDUContextLMDB index" << QByteArray::number(m_currentIndex) << "internal size mismatch:"
		389		<< m_currentValue.size() << "vs" << m_currentLen;
		390		}
		391		char *data;
		392		size_t dataLen = 0;
		393		int lz4BufLen = m_currentLen > 2 * sizeof(qint64) ? LZ4_compressBound(m_currentLen) : 0;
		394		lmdb::val value;
		395		if (lz4BufLen) {
		396		data = new char[lz4BufLen + sizeof(qint64)];
		397		// compress to an qint64-sized offset into the dest buffer; the original size will be stored in
		398		// those first 64 bits.
		399		dataLen = LZ4_compress_fast_extState(LMDB.s_lz4State, m_currentValue.constData(), &data[sizeof(qint64)],
		400		m_currentLen, lz4BufLen, 1);
		401		if (dataLen && dataLen + sizeof(qint64) < m_currentLen) {
		402		LZ4Frame frame;
		403		frame.bytes = data;
		404		frame.qint32Ptr[0] = m_currentLen;
		405		frame.qint32Ptr[1] = dataLen;
		406		value = lmdb::val(data, dataLen + sizeof(qint64));
		407		compRatioSum += double(m_currentLen) / value.size();
		408		compRatioN += 1;
		409		} else {
		410		qCDebug(LANGUAGE) << "Index" << m_currentIndex << "compression failed or useless: m_currentLen=" << m_currentLen
		411		<< "compressedLen=" << dataLen << "LZ4_compressBound=" << lz4BufLen;
		412		delete data;
		413		dataLen = 0;
		414		lz4BufLen = 0;
		415		}
		416		}
		417		if (!dataLen) {
		418		value = lmdb::val(m_currentValue.constData(), m_currentLen);
		419		uncompN += 1;
		420		}
		421		lmdb::val key(m_currentKey.constData(), m_currentKey.size());
		422		try {
		423		auto txn = lmdb::txn::begin(LMDB.handle());
		424		auto dbi = lmdb::dbi::open(txn, nullptr);
		425		try {
		426		lmdb::dbi_put(txn, dbi, key, value);
		427		txn.commit();
		428		} catch (const lmdb::error &e) {
		429		if (e.code() == MDB_MAP_FULL) {
		430		try {
		431		qCDebug(LANGUAGE) << "aborting LMDB write to grow mapsize";
		432		txn.abort();
		433		lmdb::dbi_close(LMDB.handle(), dbi);
		434		LMDB.growMapSize();
		435		commit();
		436		} catch (const lmdb::error &e) {
		437		m_errorString = lmdbxx_exception_handler(e, QStringLiteral("growing mapsize to ")
		438		+ QString::number(LMDB.s_mapSize));
		439		}
		440		} else {
		441		m_errorString = lmdbxx_exception_handler(e, QStringLiteral("committing index ")
		442		+ QByteArray::number(m_currentIndex) + " size " + QString::number(m_currentLen));
		443		}
		444		}
		445		} catch (const lmdb::error &e) {
		446		m_errorString = lmdbxx_exception_handler(e, QStringLiteral("committing index ")
		447		+ QByteArray::number(m_currentIndex) + " size " + QString::number(m_currentLen));
		448		}
		449		m_currentKey.clear();
		450		m_currentValue.clear();
		451		m_currentLen = 0;
		452		if (lz4BufLen && data) {
		453		delete data;
		454		}
		455		}
		456		}
		457
		458		bool TopDUContextLMDB::flush()
		459		{
		460		if (LMDB.instance() && m_mode != MDB_RDONLY) {
		461		try {
		462		return mdb_env_sync(LMDB.handle(), true) == MDB_SUCCESS;
		463		} catch (const lmdb::error &e) {
		464		m_errorString = lmdbxx_exception_handler(e, QStringLiteral("database flush"));
		465		}
		466		}
		467		return false;
		468		}
		469
		470		bool TopDUContextLMDB::getCurrentKeyValue()
		471		{
		472		// we only return false if a read error occurred; if a key doesn't exist
		473		// m_currentValue will remain empty.
		474		bool ret = true;
		475		if (m_currentValue.isEmpty()) {
		476		// read the key value from storage into cache
		477		try {
		478		auto rtxn = lmdb::txn::begin(LMDB.handle(), nullptr, MDB_RDONLY);
		479		auto dbi = lmdb::dbi::open(rtxn, nullptr);
		480		auto cursor = lmdb::cursor::open(rtxn, dbi);
		481		lmdb::val key(m_currentKey.constData(), m_currentKey.size());
		482		lmdb::val val {};
		483		bool found = cursor.get(key, val, MDB_SET);
		484		if (found) {
		485		bool isRaw = true;
		486		if (val.size() > sizeof(qint64)) {
		487		LZ4Frame frame;
		488		frame.bytes = val.data();
		489		int orgSize = frame.qint32Ptr[0];
		490		int compressedSize = val.size() - sizeof(qint64);
		491		if (orgSize > 0 && frame.qint32Ptr[1] == compressedSize) {
		492		// size m_currentValue so decompression can go into the final destination directly
		493		m_currentValue.resize(orgSize);
		494		if (LZ4_decompress_safe(&frame.bytes[sizeof(qint64)], m_currentValue.data(), compressedSize, orgSize) == orgSize) {
		495		isRaw = false;
		496		m_currentLen = m_currentValue.size();
		497		} else {
		498		// qCDebug(LANGUAGE) << "Index" << m_currentIndex << "failed LZ4 decompression from size" << compressedSize << "to size" << orgSize;
		499		}
		500		}
		501		}
		502		if (isRaw) {
		503		// qCDebug(LANGUAGE) << "Index" << m_currentIndex << "is uncompressed, size=" << val.size();
		504		m_currentValue = QByteArray(val.data(), val.size());
		505		m_currentLen = m_currentValue.size();
		506		}
		507		m_readCursor = 0;
		508		}
		509		cursor.close();
		510		rtxn.abort();
		511		} catch (const lmdb::error &e) {
		512		m_errorString = lmdbxx_exception_handler(e, QStringLiteral("reading index ") + QByteArray::number(m_currentIndex));
		513		ret = false;
		514		}
		515		}
		516		return ret;
		517		}
		518
		519		bool TopDUContextLMDB::isValid()
		520		{
		521		return LMDB.instance();
		522		}
		523
		524		bool TopDUContextLMDB::currentKeyExists()
		525		{
		526		return exists(m_currentKey);
		527		}
		528
		529		bool TopDUContextLMDB::exists(const QByteArray &key)
		530		{
		531		if (LMDB.instance()) {
		532		try {
		533		auto rtxn = lmdb::txn::begin(LMDB.handle(), nullptr, MDB_RDONLY);
		534		auto dbi = lmdb::dbi::open(rtxn, nullptr);
		535		auto cursor = lmdb::cursor::open(rtxn, dbi);
		536		lmdb::val k(key.constData(), key.size());
		537		bool ret = cursor.get(k, nullptr, MDB_SET);
		538		cursor.close();
		539		rtxn.abort();
		540		return ret;
		541		} catch (const lmdb::error &e) {
		542		lmdbxx_exception_handler(e, QStringLiteral("checking for index") + key);
		543		}
		544		}
		545		return false;
		546		}
		547
		548		QString TopDUContextLMDB::fileName() const
		549		{
		550		return TopDUContextDynamicData::basePath() + "data.mdb" + ":#" + QByteArray::number(m_currentIndex);
		551		}
		552
		553		bool TopDUContextLMDB::exists(uint topContextIndex)
		554		{
		555		return exists(indexKey(topContextIndex));
		556		}
		557
		558		bool TopDUContextLMDB::remove(uint topContextIndex)
		559		{
		560		if (LMDB.instance()) {
		561		const auto key = indexKey(topContextIndex);
		562		lmdb::val k {key.constData(), static_cast<size_t>(key.size())};
		563		try {
		564		auto txn = lmdb::txn::begin(LMDB.handle());
		565		auto dbi = lmdb::dbi::open(txn, nullptr);
		566		bool ret = lmdb::dbi_del(txn, dbi, k, nullptr);
		567		txn.commit();
		568		// also remove the file if it (still) exists
		569		QFile::remove(TopDUContextDynamicData::pathForTopContext(topContextIndex));
		570		return ret;
		571		} catch (const lmdb::error &e) {
		572		lmdbxx_exception_handler(e, QStringLiteral("removing index %1").arg(topContextIndex));
		573		}
		574		}
		575		return false;
		576		}
		577		#endif
		578
		579		#ifdef KDEV_TOPCONTEXTS_USE_LEVELDB
		580		// LevelDB storage backend
		581		static QString leveldb_exception_handler(const std::exception &e, const QString &operation)
		582		{
		583		const QString msg = QStringLiteral("LevelDB exception in \"%1\": %2").arg(operation).arg(e.what());
		584		qCWarning(LANGUAGE) << msg;
		585		if (qEnvironmentVariableIsSet("KDEV_TOPCONTEXTS_STORE_FAILURE_ABORT")) {
		586		qFatal(msg.toLatin1());
		587		}
		588		return msg;
		589		}
		590
		591		// our keys are uint values and stored as uint* in the original QByteArray m_currentKey
		592		// use that knowledge to avoid sorting lexographically.
		593		class QuickSortingComparator : public leveldb::Comparator
		594		{
		595		public:
		596		// Comparison function that doesn't result in any sorting
		597		int Compare(const leveldb::Slice &a, const leveldb::Slice &b) const
		598		{
		599		const uint ia = reinterpret_cast<const uint >(a.data());
		600		const uint ib = reinterpret_cast<const uint >(b.data());
		601		return ia - ib;
		602		}
		603
		604		const char *Name() const
		605		{
		606		return "QuickSortingComparator";
		607		}
		608		void FindShortestSeparator(std::string *, const leveldb::Slice &) const {}
		609		void FindShortSuccessor(std::string *) const {}
		610		};
		611		static QuickSortingComparator leveldbComparator;
		612
		613		// there is exactly 1 topcontexts directory per session, so we can make do with a single
		614		// global static LevelDB instance which is not exported at all (= no need to include
		615		// db.h in our own headerfile).
		616
		617		class LevelDBHook
		618		{
		619		public:
		620		~LevelDBHook()
		621		{
		622		if (s_levelDB) {
		623		delete s_levelDB;
		624		}
		625		}
		626
		627		static const leveldb::Status init()
		628		{
		629		leveldb::Options options;
		630		options.create_if_missing = true;
		631		options.comparator = &leveldbComparator;
		632		int attempts = 0;
		633		leveldb::Status status;
		634		do {
		635		attempts += 1;
		636		leveldb::Status status = leveldb::DB::Open(options, TopDUContextDynamicData::basePath().toStdString(), &s_levelDB);
		637		if (!status.ok()) {
		638		s_levelDB = nullptr;
		639		if (status.IsInvalidArgument()) {
		640		// retry opening a fresh store
		641		leveldb::DestroyDB(TopDUContextDynamicData::basePath().toStdString(), options);
		642		}
		643		}
		644		} while (!s_levelDB && attempts < 2);
		645		return status;
		646		}
		647
		648		inline leveldb::DB *instance()
		649		{
		650		return s_levelDB;
		651		}
		652
		653		static leveldb::DB *s_levelDB;
		654		};
		655		static LevelDBHook levelDB;
		656
		657		leveldb::DB *LevelDBHook::s_levelDB = nullptr;
		658		uint TopDUContextLevelDB::s_DbRefCount = 0;
		659
		660		TopDUContextLevelDB::TopDUContextLevelDB(uint topContextIndex)
		661		{
		662		m_currentIndex = topContextIndex;
		663		if (!levelDB.instance() && Q_LIKELY(QFileInfo(TopDUContextDynamicData::basePath()).isWritable())) {
		664		leveldb::Status status = LevelDBHook::init();
		665		if (!status.ok()) {
		666		m_errorString = QStringLiteral("Error opening LevelDB database:") + QString::fromStdString(status.ToString());
		667		qCWarning(LANGUAGE) << m_errorString;
		668		}
		669		}
		670		if (levelDB.instance()) {
		671		m_currentKey = indexKey(&m_currentIndex);
		672		s_DbRefCount += 1;
		673		}
		674		m_currentLen = -1;
		675		// the remaining member vars are initialised elsewhere intentionally.
		676		}
		677
		678		TopDUContextLevelDB::~TopDUContextLevelDB()
		679		{
		680		if (levelDB.instance()) {
		681		s_DbRefCount -= 1;
		682		if (s_DbRefCount <= 0) {
		683		// optimisation: don't delete the global DB handle; too many TopDUContextLevelDB
		684		// instances are created too frequently that are deleted immediately after a
		685		// single use.
		686		s_DbRefCount = 0;
		687		}
		688		} else {
		689		s_DbRefCount = 0;
		690		}
		691		}
		692
		693		bool TopDUContextLevelDB::open(QIODevice::OpenMode mode)
		694		{
		695		return TopDUContextDB::open(mode, QStringLiteral("LevelDB"));
		696		}
		697
		698		void TopDUContextLevelDB::commit()
		699		{
		700		if (isValid() && m_mode != MDB_RDONLY) {
		701		if (m_currentValue.size() != m_currentLen) {
		702		// m_currentLen is the true size
		703		qCDebug(LANGUAGE) << "TopDUContextLevelDB index" << QByteArray::number(m_currentIndex) << "internal size mismatch:"
		704		<< m_currentValue.size() << "vs" << m_currentLen;
		705		}
		706		leveldb::Status status;
		707		leveldb::Slice key(m_currentKey.constData(), m_currentKey.size());
		708		leveldb::Slice value(m_currentValue.constData(), m_currentLen);
		709		try {
		710		status = levelDB.instance()->Put(leveldb::WriteOptions(), key, value);
		711		} catch (const std::exception &e) {
		712		m_errorString = leveldb_exception_handler(e, QStringLiteral("committing value for ") + QByteArray::number(m_currentIndex));
		713		qCWarning(LANGUAGE) << m_errorString;
		714		}
		715		if (!status.ok()) {
		716		m_errorString = QStringLiteral("Error committing index ")
		717		+ QByteArray::number(m_currentIndex) + " size " + QString::number(m_currentLen)
		718		+ QStringLiteral(": ") + QString::fromStdString(status.ToString());
		719		qCWarning(LANGUAGE) << m_errorString;
		720		}
		721		m_currentKey.clear();
		722		m_currentValue.clear();
		723		m_currentLen = 0;
		724		}
		725		}
		726
		727		bool TopDUContextLevelDB::flush()
		728		{
		729		if (isValid() && m_mode != MDB_RDONLY) {
		730		try {
		731		levelDB.instance()->CompactRange(nullptr, nullptr);
		732		return true;
		733		} catch (const std::exception &e) {
		734		m_errorString = leveldb_exception_handler(e, QStringLiteral("database flush (CompactRange)"));
		735		qCWarning(LANGUAGE) << m_errorString;
		736		}
		737		}
		738		return false;
		739		}
		740
		741		bool TopDUContextLevelDB::getCurrentKeyValue()
		742		{
		743		// we only return false if a read error occurred; if a key doesn't exist
		744		// m_currentValue will remain empty.
		745		bool ret = true;
		746		if (m_currentValue.isEmpty()) {
		747		// read the key value from storage into cache
		748		leveldb::Slice key(m_currentKey.constData(), m_currentKey.size());
		749		std::string value;
		750		leveldb::Status status = levelDB.instance()->Get(leveldb::ReadOptions(), key, &value);
		751		if (status.ok()) {
		752		m_currentValue = QByteArray(value.data(), value.size());
		753		m_currentLen = value.size();
		754		m_readCursor = 0;
		755		} else if (status.IsCorruption() \|\| status.IsIOError() \|\| status.IsNotSupportedError() \|\| status.IsInvalidArgument()) {
		756		ret = false;
		757		}
		758		}
		759		return ret;
		760		}
		761
		762		bool TopDUContextLevelDB::isValid()
		763		{
		764		return levelDB.instance();
		765		}
		766
		767		bool TopDUContextLevelDB::currentKeyExists()
		768		{
		769		return exists(m_currentKey);
		770		}
		771
		772		bool TopDUContextLevelDB::exists(const QByteArray &key)
		773		{
		774		if (levelDB.instance()) {
		775		std::string value;
		776		leveldb::Slice k(key.constData(), key.size());
		777		leveldb::Status status = levelDB.instance()->Get(leveldb::ReadOptions(), k, &value);
		778		if (!status.ok() && !status.IsNotFound()) {
		779		qCWarning(LANGUAGE) << QStringLiteral("Error checking for index") + key
		780		+ QStringLiteral(": ") + QString::fromStdString(status.ToString());
		781		return false;
		782		}
		783		return status.ok();
		784		}
		785		return false;
		786		}
		787
		788		bool TopDUContextLevelDB::exists(uint topContextIndex)
		789		{
		790		return exists(indexKey(&topContextIndex));
		791		}
		792
		793		bool TopDUContextLevelDB::remove(uint topContextIndex)
		794		{
		795		if (levelDB.instance()) {
		796		const auto key = indexKey(&topContextIndex);
		797		leveldb::Slice k(key.constData(), key.size());
		798		leveldb::Status status = levelDB.instance()->Delete(leveldb::WriteOptions(), k);
		799		if (!status.ok()) {
		800		qCWarning(LANGUAGE) << QStringLiteral("Error removing index %1").arg(topContextIndex)
		801		+ QStringLiteral(": ") + QString::fromStdString(status.ToString());
		802		return false;
		803		}
		804		// also remove the file if it (still) exists
		805		QFile::remove(TopDUContextDynamicData::pathForTopContext(topContextIndex));
		806		return true;
		807		}
		808		return false;
		809		}
		810
		811		#endif // KDEV_TOPCONTEXTS_USE_LEVELDB
		812
		813		#ifdef KDEV_TOPCONTEXTS_USE_KYOTO
		814		using namespace kyotocabinet;
		815
		816		static QString kyotocabinet_exception_handler(const std::exception &e, const QString &operation)
		817		{
		818		const QString msg = QStringLiteral("KyotoCabinet exception in \"%1\": %2").arg(operation).arg(e.what());
		819		qCWarning(LANGUAGE) << msg;
		820		if (qEnvironmentVariableIsSet("KDEV_TOPCONTEXTS_STORE_FAILURE_ABORT")) {
		821		qFatal(msg.toLatin1());
		822		}
		823		return msg;
		824		}
		825
		826		// there is exactly 1 topcontexts directory per session, which is fine because we can only use a single
		827		// global static KyotoCabinet instance anyway. Wrap it in a local class which is not exported at all
		828		// (= no need to include kcpolydb.h in our own headerfile).
		829
		830		class KyotoCabinetHook
		831		{
		832		public:
		833		~KyotoCabinetHook()
		834		{
		835		if (s_kyotoCab) {
		836		try {
		837		if (!s_kyotoCab->close()) {
		838		qCWarning(LANGUAGE) << "Error closing KyotoCabinet database:" << errorString();
		839		}
		840		delete s_kyotoCab;
		841		} catch (const std::exception &e) {
		842		qCWarning(LANGUAGE) << kyotocabinet_exception_handler(e, QStringLiteral("closing cabinet.kch"));
		843		// don't delete s_kyotoCab if something went wrong closing the DB
		844		}
		845		}
		846		}
		847
		848		static bool init()
		849		{
		850		s_kyotoCab = new PolyDB;
		851		bool ok = false;
		852		s_errorString.clear();
		853		try {
		854		int attempts = 0;
		855		uint32_t flags = PolyDB::OWRITER \| PolyDB::OCREATE;
		856		do {
		857		attempts += 1;
		858		// for logging, append
		859		// #log=+#logkinds=debug
		860		// for compression (ZLIB deflate), append
		861		// #opts=sc#zcomp=def
		862		ok = s_kyotoCab->open(TopDUContextDynamicData::basePath().toStdString() + "cabinet.kch#opts=sc#zcomp=lzo", flags);
		863		if (!ok && s_kyotoCab->error().code() == BasicDB::Error::INVALID) {
		864		// try by recreating the database
		865		flags = PolyDB::OWRITER \| PolyDB::OTRUNCATE;
		866		}
		867		} while (!ok && attempts < 2);
		868		} catch (const std::exception &e) {
		869		s_errorString = kyotocabinet_exception_handler(e, QStringLiteral("opening cabinet.kch"));
		870		ok = false;
		871		}
		872		if (!ok) {
		873		if (s_errorString.isEmpty()) {
		874		s_errorString = QStringLiteral("Error opening KyotoCabinet database:")
		875		+ errorString();
		876		}
		877		delete s_kyotoCab;
		878		s_kyotoCab = nullptr;
		879		}
		880		return s_kyotoCab;
		881		}
		882
		883		inline PolyDB *instance()
		884		{
		885		return s_kyotoCab;
		886		}
		887
		888		static inline QString errorString()
		889		{
		890		return QString::fromStdString(s_kyotoCab->error().name());
		891		}
		892
		893		static PolyDB *s_kyotoCab;
		894		static QString s_errorString;
		895		};
		896		static KyotoCabinetHook kyotoCabinet;
		897
		898		PolyDB *KyotoCabinetHook::s_kyotoCab = nullptr;
		899		QString KyotoCabinetHook::s_errorString;
		900
		901		uint TopDUContextKyotoCabinet::s_DbRefCount = 0;
		902
		903		TopDUContextKyotoCabinet::TopDUContextKyotoCabinet(uint topContextIndex)
		904		{
		905		m_currentIndex = topContextIndex;
		906		if (!kyotoCabinet.instance() && Q_LIKELY(QFileInfo(TopDUContextDynamicData::basePath()).isWritable())) {
		907		if (!kyotoCabinet.init()) {
		908		m_errorString = kyotoCabinet.s_errorString;
		909		qCWarning(LANGUAGE) << m_errorString;
		910		}
		911		}
		912		if (kyotoCabinet.instance()) {
		913		m_currentKey = indexKey(&m_currentIndex);
		914		s_DbRefCount += 1;
		915		}
		916		m_currentLen = -1;
		917		// the remaining member vars are initialised elsewhere intentionally.
		918		}
		919
		920		TopDUContextKyotoCabinet::~TopDUContextKyotoCabinet()
		921		{
		922		if (kyotoCabinet.instance()) {
		923		s_DbRefCount -= 1;
		924		if (s_DbRefCount <= 0) {
		925		// optimisation: don't delete the global DB handle; too many TopDUContextKyotoCabinet
		926		// instances are created too frequently that are deleted immediately after a
		927		// single use.
		928		s_DbRefCount = 0;
		929		}
		930		#ifdef DEBUG
		931		kyotoCabinet.instance()->synchronize(s_DbRefCount == 0);
		932		#endif
		933		} else {
		934		s_DbRefCount = 0;
		935		}
		936		}
		937
		938		QString TopDUContextKyotoCabinet::fileName() const
		939		{
		940		return TopDUContextDynamicData::basePath() + "cabinet.kch:#" + QByteArray::number(m_currentIndex);
		941		}
		942
		943		bool TopDUContextKyotoCabinet::open(QIODevice::OpenMode mode)
		944		{
		945		return TopDUContextDB::open(mode, QStringLiteral("KyotoCabinet"));
		946		}
		947
		948		void TopDUContextKyotoCabinet::commit()
		949		{
		950		if (isValid() && m_mode != MDB_RDONLY) {
		951		if (m_currentValue.size() != m_currentLen) {
		952		// m_currentLen is the true size
		953		qCDebug(LANGUAGE) << "TopDUContextKyotoCabinet index" << QByteArray::number(m_currentIndex) << "internal size mismatch:"
		954		<< m_currentValue.size() << "vs" << m_currentLen;
		955		}
		956		try {
		957		if (!kyotoCabinet.instance()->set(m_currentKey.constData(), m_currentKey.size(),
		958		m_currentValue.constData(), m_currentLen)) {
		959		m_errorString = QStringLiteral("Error committing index ")
		960		+ QByteArray::number(m_currentIndex) + " size " + QString::number(m_currentLen)
		961		+ QStringLiteral(": ") + kyotoCabinet.errorString();
		962		qCWarning(LANGUAGE) << m_errorString;
		963		}
		964		} catch (const std::exception &e) {
		965		m_errorString = kyotocabinet_exception_handler(e, QStringLiteral("committing value for ") + QByteArray::number(m_currentIndex));
		966		qCWarning(LANGUAGE) << m_errorString;
		967		}
		968		m_currentKey.clear();
		969		m_currentValue.clear();
		970		m_currentLen = 0;
		971		}
		972		}
		973
		974		bool TopDUContextKyotoCabinet::flush()
		975		{
		976		if (isValid() && m_mode != MDB_RDONLY) {
		977		try {
		978		return kyotoCabinet.instance()->synchronize(true);
		979		} catch (const std::exception &e) {
		980		m_errorString = kyotocabinet_exception_handler(e, QStringLiteral("database flush"));
		981		qCWarning(LANGUAGE) << m_errorString;
		982		}
		983		}
		984		return false;
		985		}
		986
		987		bool TopDUContextKyotoCabinet::getCurrentKeyValue()
		988		{
		989		// we only return false if a read error occurred; if a key doesn't exist
		990		// m_currentValue will remain empty.
		991		bool ret = true;
		992		if (m_currentValue.isEmpty()) {
		993		// read the key value from storage into cache
		994		size_t size;
		995		try {
		996		const char *value = kyotoCabinet.instance()->get(m_currentKey.constData(), m_currentKey.size(), &size);
		997		if (value) {
		998		m_currentValue = QByteArray(value, size);
		999		m_currentLen = m_currentValue.size();
		1000		m_readCursor = 0;
		1001		delete[] value;
		1002		} else {
		1003		qCWarning(LANGUAGE) << "read NULL for index" << m_currentIndex << "; exists=" << currentKeyExists();
		1004		}
		1005		} catch (const std::exception &e) {
		1006		m_errorString = kyotocabinet_exception_handler(e, QStringLiteral("reading value for ") + QByteArray::number(m_currentIndex));
		1007		qCWarning(LANGUAGE) << m_errorString;
		1008		ret = false;
		1009		}
		1010		}
		1011		return ret;
		1012		}
		1013
		1014		bool TopDUContextKyotoCabinet::isValid()
		1015		{
		1016		return kyotoCabinet.instance();
		1017		}
		1018
		1019		bool TopDUContextKyotoCabinet::currentKeyExists()
		1020		{
		1021		return exists(m_currentKey);
		1022		}
		1023
		1024		bool TopDUContextKyotoCabinet::exists(const QByteArray &key)
		1025		{
		1026		if (kyotoCabinet.instance()) {
		1027		try {
		1028		bool found = kyotoCabinet.instance()->check(key.constData(), key.size()) >= 0;
		1029		if (!found && kyotoCabinet.instance()->error().code() != BasicDB::Error::NOREC) {
		1030		qCWarning(LANGUAGE) << QStringLiteral("Error checking for index") + key
		1031		+ QStringLiteral(": ") + kyotoCabinet.errorString();
		1032		}
		1033		return found;
		1034		} catch (const std::exception &e) {
		1035		qCWarning(LANGUAGE) << kyotocabinet_exception_handler(e, QStringLiteral("checking presence of ") + key);
		1036		}
		1037		}
		1038		return false;
		1039		}
		1040
		1041		bool TopDUContextKyotoCabinet::exists(uint topContextIndex)
		1042		{
		1043		return exists(indexKey(&topContextIndex));
		1044		}
		1045
		1046		bool TopDUContextKyotoCabinet::remove(uint topContextIndex)
		1047		{
		1048		if (kyotoCabinet.instance()) {
		1049		const auto key = indexKey(&topContextIndex);
		1050		try {
		1051		bool ret = kyotoCabinet.instance()->remove(key.constData(), key.size());
		1052		if (!ret) {
		1053		qCWarning(LANGUAGE) << QStringLiteral("Error removing index %1").arg(topContextIndex)
		1054		+ QStringLiteral(": ") + kyotoCabinet.errorString();
		1055		}
		1056		// also remove the file if it (still) exists
		1057		QFile::remove(TopDUContextDynamicData::pathForTopContext(topContextIndex));
		1058		return ret;
		1059		} catch (const std::exception &e) {
		1060		qCWarning(LANGUAGE) << kyotocabinet_exception_handler(e, QStringLiteral("removing ") + QByteArray::number(topContextIndex));
		1061		}
		1062		}
		1063		return false;
		1064		}
		1065		#endif // KDEV_TOPCONTEXTS_USE_KYOTO
		1066
		1067		#endif // KDEV_TOPCONTEXTS_USE_FILES
		1068
		1069		// TopDUContextFile : thin wrapper around the QFile API needed for TopDUContexts
		1070		// so TopDUContextLMDB can be used as a drop-in replacement instead of this class.
		1071		TopDUContextFile::TopDUContextFile(uint topContextIndex)
		1072		: QFile(TopDUContextDynamicData::pathForTopContext(topContextIndex))
		1073		{
		1074		}
		1075
		1076		bool TopDUContextFile::exists(uint topContextIndex)
		1077		{
		1078		return QFile::exists(TopDUContextDynamicData::pathForTopContext(topContextIndex));
		1079		}
		1080
		1081		bool TopDUContextFile::remove(uint topContextIndex)
		1082		{
		1083		return QFile::remove(TopDUContextDynamicData::pathForTopContext(topContextIndex));
		1084		}
		1085
		1086		void TopDUContextFile::commit()
		1087		{
		1088		QFile::close();
		1089		}